From b554bdd28970f053d1421e4c39afad73fd79d2e4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 6 Nov 2026 21:49:20 +0000 Subject: [PATCH 1/2] Merge pull request #128217 from groeneai/fix-partition-by-stateful-function-spans-chunks Evaluate the `PARTITION BY` expression of file and object storage writes once per INSERT, not once per block Conflicts committed as-is; the next commit resolves them: - src/Storages/IPartitionStrategy.cpp (both modified) - src/Storages/IPartitionStrategy.h (both modified) --- src/Storages/IPartitionStrategy.cpp | 118 +++++++++++++++--- src/Storages/IPartitionStrategy.h | 18 ++- src/Storages/PartitionedSink.cpp | 3 +- src/Storages/PartitionedSink.h | 1 + ...y_stateful_function_spans_chunks.reference | 11 ++ ...tion_by_stateful_function_spans_chunks.sql | 25 ++++ 6 files changed, 159 insertions(+), 17 deletions(-) create mode 100644 tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.reference create mode 100644 tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.sql diff --git a/src/Storages/IPartitionStrategy.cpp b/src/Storages/IPartitionStrategy.cpp index 01c85f481427c..92d6ce0d68641 100644 --- a/src/Storages/IPartitionStrategy.cpp +++ b/src/Storages/IPartitionStrategy.cpp @@ -241,6 +241,16 @@ IPartitionStrategy::getPartitionExpressionActions(ASTPtr & expression_ast) const return result; } +ColumnPtr IPartitionStrategy::computePartitionKey( + const Chunk & chunk, const PartitionExpressionActionsAndColumnName & partition_key_actions) const +{ + Block block_with_partition_by_expr = sample_block.cloneWithoutColumns(); + block_with_partition_by_expr.setColumns(chunk.getColumns()); + partition_key_actions.actions->execute(block_with_partition_by_expr); + + return block_with_partition_by_expr.getByName(partition_key_actions.column_name).column; +} + std::shared_ptr PartitionStrategyFactory::get(StrategyType strategy, ASTPtr partition_by, const NamesAndTypesList & partition_columns, @@ -298,18 +308,12 @@ WildcardPartitionStrategy::WildcardPartitionStrategy(KeyDescription partition_ke cacheDeterministicActions(cached_result, actions_with_column); } -ColumnPtr WildcardPartitionStrategy::computePartitionKey(const Chunk & chunk) const +IPartitionStrategy::PartitionExpressionActionsAndColumnName WildcardPartitionStrategy::getPartitionKeyActions() const { - auto actions_with_column = getCachedOrBuildActions( + return getCachedOrBuildActions( cached_result, *this, [&] { return buildToStringPartitionAST(partition_key_description.definition_ast); }); - - Block block_with_partition_by_expr = sample_block.cloneWithoutColumns(); - block_with_partition_by_expr.setColumns(chunk.getColumns()); - actions_with_column.actions->execute(block_with_partition_by_expr); - - return block_with_partition_by_expr.getByName(actions_with_column.column_name).column; } ColumnPtr WildcardPartitionStrategy::computePartitionKey(Block & block) const @@ -348,18 +352,102 @@ HiveStylePartitionStrategy::HiveStylePartitionStrategy( cacheDeterministicActions(cached_result, actions_with_column); } +<<<<<<< HEAD ColumnPtr HiveStylePartitionStrategy::computePartitionKey(const Chunk & chunk) const +======= +std::string HiveStylePartitionStrategy::getPathForRead(const std::string & prefix) { - auto actions_with_column = getCachedOrBuildActions( + /// Match every file extension registered for the format, not only the lowercased format + /// name: for most formats the name is not the extension real files carry (`JSONEachRow` + /// files are named `.jsonl` / `.ndjson`, `CSVWithNames` files are named `.csv`), and a + /// glob built from the format name alone silently matches nothing over such a lake. + /// The lowercased format name is always one of the registered extensions, so the files + /// ClickHouse itself writes (see getPathForWrite) keep matching. + const auto extensions = FormatFactory::instance().getFileExtensionsForFormat(file_format); + + /// The glob is matched against the object key as it is stored, before anything decompresses it + /// (`GlobIterator` filters the listing, while the compression method is derived much later, in + /// `ReadBufferIterator`), so a compressed lake of `key=1/data.jsonl.gz` objects is invisible to + /// a glob of bare extensions. The glob has to accept exactly the names the reader would accept, + /// which is the rule of `chooseCompressionMethod`: + /// - `auto` (the default): the file name decides, so spell out every suffix it recognizes; + /// - an explicit codec: the file name is ignored, so a `gzip` lake of `data.jsonl.custom` objects + /// is as readable as one of `data.jsonl.gz` objects, and anything after the format extension + /// must match. The bare extension is kept too, because an explicit codec also applies to + /// files named without any suffix - and that is what `getPathForWrite` produces; + /// - `none`: the files carry no compression layer, so a suffix after the format extension is + /// not a compression spelling to accept but foreign data (`.parquet.crc` sidecars and the + /// like), and only the bare extensions match. + Strings alternatives = extensions; + + std::string compression_hint = compression_method; + boost::algorithm::to_lower(compression_hint); + + if (compression_hint.empty() || compression_hint == "auto") + { + for (const auto & compression_suffix : getFileSuffixesForCompressionMethodHint(compression_method)) + for (const auto & extension : extensions) + alternatives.push_back(extension + "." + compression_suffix); + } + else if (compression_hint != "none") + { + /// A misspelled codec is not validated here: this runs on `ATTACH` and at server startup too, + /// where throwing would make existing metadata unloadable. `CREATE TABLE` rejects it in + /// `StorageObjectStorageConfiguration::initPartitionStrategy`, and a table attached from older + /// metadata keeps the suffix alternatives, so its reads reach `chooseCompressionMethod` and + /// fail loudly with `Unknown compression method` instead of silently matching nothing. + /// + /// The suffix is arbitrary, but it has to be a suffix: `.csv.*` and not `.csv*`, which is a + /// prefix match on the extension and would hand the `data.csvwithnames.gz` files of a sibling + /// `CSVWithNames` lake to the `CSV` parser (the reader trusts the explicit codec and never looks + /// at the name). The format boundary is the `.` after the extension. + for (const auto & extension : extensions) + alternatives.push_back(extension + ".*"); + } + + if (alternatives.size() == 1) + return prefix + "**." + alternatives.front(); + + return prefix + "**.{" + boost::algorithm::join(alternatives, ",") + "}"; +} + +std::string HiveStylePartitionStrategy::getPathForWrite( + const std::string & prefix, + const std::string & partition_key) +{ + std::string path; + + if (!prefix.empty()) + { + path += prefix; + if (path.back() != '/') + { + path += '/'; + } + } + + /// Not adding '/' because buildExpressionHive() always adds a trailing '/' + path += partition_key; + + /* + * File extension is toLower(format) + * This isn't ideal, but I guess multiple formats can be specified and introduced. + * So I think it is simpler to keep it this way. + * + * Or perhaps implement something like `IInputFormat::getFileExtension()` + */ + path += std::to_string(generateSnowflakeID()) + "." + Poco::toLower(file_format); + + return path; +} + +IPartitionStrategy::PartitionExpressionActionsAndColumnName HiveStylePartitionStrategy::getPartitionKeyActions() const +>>>>>>> 690d3be35e1 (Merge pull request #128217 from groeneai/fix-partition-by-stateful-function-spans-chunks) +{ + return getCachedOrBuildActions( cached_result, *this, [&] { return buildHivePartitionAST(partition_key_description.definition_ast, getPartitionColumns()); }); - - Block block_with_partition_by_expr = sample_block.cloneWithoutColumns(); - block_with_partition_by_expr.setColumns(chunk.getColumns()); - actions_with_column.actions->execute(block_with_partition_by_expr); - - return block_with_partition_by_expr.getByName(actions_with_column.column_name).column; } ColumnPtr HiveStylePartitionStrategy::computePartitionKey(Block & block) const diff --git a/src/Storages/IPartitionStrategy.h b/src/Storages/IPartitionStrategy.h index 1378762c6911d..901db76eef5f0 100644 --- a/src/Storages/IPartitionStrategy.h +++ b/src/Storages/IPartitionStrategy.h @@ -27,7 +27,11 @@ struct IPartitionStrategy virtual ~IPartitionStrategy() = default; - virtual ColumnPtr computePartitionKey(const Chunk & chunk) const = 0; + /// Rebuilt on every call if non-deterministic. A sink calls it once and reuses the result for all its chunks, + /// so a stateful function keeps its state across the chunks of one INSERT. + virtual PartitionExpressionActionsAndColumnName getPartitionKeyActions() const = 0; + + ColumnPtr computePartitionKey(const Chunk & chunk, const PartitionExpressionActionsAndColumnName & partition_key_actions) const; virtual ColumnPtr computePartitionKey(Block & block) const = 0; @@ -91,9 +95,15 @@ struct WildcardPartitionStrategy : IPartitionStrategy { WildcardPartitionStrategy(KeyDescription partition_key_description_, const Block & sample_block_, ContextPtr context_); +<<<<<<< HEAD ColumnPtr computePartitionKey(const Chunk & chunk) const override; ColumnPtr computePartitionKey(Block & block) const override; +======= + PartitionExpressionActionsAndColumnName getPartitionKeyActions() const override; + std::string getPathForRead(const std::string & prefix) override; + std::string getPathForWrite(const std::string & prefix, const std::string & partition_key) override; +>>>>>>> 690d3be35e1 (Merge pull request #128217 from groeneai/fix-partition-by-stateful-function-spans-chunks) }; /* @@ -110,9 +120,15 @@ struct HiveStylePartitionStrategy : IPartitionStrategy const std::string & file_format_, bool partition_columns_in_data_file_); +<<<<<<< HEAD ColumnPtr computePartitionKey(const Chunk & chunk) const override; ColumnPtr computePartitionKey(Block & block) const override; +======= + PartitionExpressionActionsAndColumnName getPartitionKeyActions() const override; + std::string getPathForRead(const std::string & prefix) override; + std::string getPathForWrite(const std::string & prefix, const std::string & partition_key) override; +>>>>>>> 690d3be35e1 (Merge pull request #128217 from groeneai/fix-partition-by-stateful-function-spans-chunks) ColumnRawPtrs getFormatChunkColumns(const Chunk & chunk) override; Block getFormatHeader() override; diff --git a/src/Storages/PartitionedSink.cpp b/src/Storages/PartitionedSink.cpp index fbc41f6be4d39..f44539eb5adaf 100644 --- a/src/Storages/PartitionedSink.cpp +++ b/src/Storages/PartitionedSink.cpp @@ -34,6 +34,7 @@ PartitionedSink::PartitionedSink( , sink_creator(sink_creator_) , context(context_) , source_header(source_header_) + , partition_key_actions(partition_strategy->getPartitionKeyActions()) { } @@ -52,7 +53,7 @@ SinkPtr PartitionedSink::getSinkForPartitionKey(std::string_view partition_key) void PartitionedSink::consume(Chunk & source_chunk) { - const ColumnPtr partition_by_result_column = partition_strategy->computePartitionKey(source_chunk); + const ColumnPtr partition_by_result_column = partition_strategy->computePartitionKey(source_chunk, partition_key_actions); /// Not all columns are serialized using the format writer (e.g, hive partitioning stores partition columns in the file path) const auto columns_to_consume = partition_strategy->getFormatChunkColumns(source_chunk); diff --git a/src/Storages/PartitionedSink.h b/src/Storages/PartitionedSink.h index 1f612076db92f..9bf07defb3fc2 100644 --- a/src/Storages/PartitionedSink.h +++ b/src/Storages/PartitionedSink.h @@ -51,6 +51,7 @@ class PartitionedSink : public SinkToStorage std::shared_ptr sink_creator; ContextPtr context; SharedHeader source_header; + IPartitionStrategy::PartitionExpressionActionsAndColumnName partition_key_actions; absl::flat_hash_map partition_id_to_sink; HashMapWithSavedHash partition_id_to_chunk_index; diff --git a/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.reference b/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.reference new file mode 100644 index 0000000000000..61a6ea609e207 --- /dev/null +++ b/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.reference @@ -0,0 +1,11 @@ +05459_rn_0.csv 1000 0 999 +05459_rn_1.csv 1000 1000 1999 +05459_rn_2.csv 1000 2000 2999 +05459_rn_3.csv 1000 3000 3999 +05459_rn_4.csv 1000 4000 4999 +05459_eng_0.csv 1000 10000 10999 +05459_eng_1.csv 1000 11000 11999 +05459_eng_2.csv 1000 2000 2999 +05459_eng_3.csv 1000 3000 3999 +05459_eng_4.csv 1000 4000 4999 +1 5000 diff --git a/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.sql b/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.sql new file mode 100644 index 0000000000000..54420711c58df --- /dev/null +++ b/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.sql @@ -0,0 +1,25 @@ +-- Tags: no-fasttest +-- Tag no-fasttest: uses S3 + +SET max_block_size = 1000, max_insert_threads = 1, engine_file_truncate_on_insert = 1; + +-- file(): the counter spans the five 1000-row chunks +INSERT INTO FUNCTION file(currentDatabase() || '/05459_rn_{_partition_id}.csv', 'CSV', 'n UInt64') +PARTITION BY intDiv(rowNumberInAllBlocks(), 1000) +SELECT number FROM numbers(5000); +SELECT _file, count(), min(n), max(n) FROM file(currentDatabase() || '/05459_rn_*.csv', 'CSV', 'n UInt64') GROUP BY _file ORDER BY _file; + +-- S3 engine (strategy shared by all INSERTs into the table): spans chunks, starts anew with the next INSERT +CREATE TABLE t_05459 (n UInt64) +ENGINE = S3(s3_conn, filename = currentDatabase() || '/05459_eng_{_partition_id}.csv', format = CSV, partition_strategy = 'wildcard') +PARTITION BY intDiv(rowNumberInAllBlocks(), 1000); +INSERT INTO t_05459 SELECT number FROM numbers(5000) SETTINGS s3_truncate_on_insert = 1, s3_create_new_file_on_insert = 0; +INSERT INTO t_05459 SELECT number + 10000 FROM numbers(2000) SETTINGS s3_truncate_on_insert = 1, s3_create_new_file_on_insert = 0; +SELECT _file, count(), min(n), max(n) FROM s3(s3_conn, filename = currentDatabase() || '/05459_eng_*.csv', format = CSV, structure = 'n UInt64') GROUP BY _file ORDER BY _file; +DROP TABLE t_05459; + +-- now64() is evaluated once per INSERT, not once per chunk +INSERT INTO FUNCTION file(currentDatabase() || '/05459_now_{_partition_id}.csv', 'CSV', 'n UInt64') +PARTITION BY toUnixTimestamp64Nano(now64(9)) +SELECT number FROM numbers(5000); +SELECT count(DISTINCT _file), count() FROM file(currentDatabase() || '/05459_now_*.csv', 'CSV', 'n UInt64'); From d55166491420e9d85b3f4c60623ac536ec92ef21 Mon Sep 17 00:00:00 2001 From: blau-ai <302218339+blau-ai@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:52:30 +0000 Subject: [PATCH 2/2] Resolve conflicts in backport of #128217 Dropped the upstream `getPathForRead`/`getPathForWrite` hunks, which don't exist on antalya-26.6, and kept a one-argument `computePartitionKey(Chunk)` for existing callers. --- src/Storages/IPartitionStrategy.cpp | 90 ----------------------------- src/Storages/IPartitionStrategy.h | 19 ++---- 2 files changed, 5 insertions(+), 104 deletions(-) diff --git a/src/Storages/IPartitionStrategy.cpp b/src/Storages/IPartitionStrategy.cpp index 92d6ce0d68641..3b57f9cfcf56c 100644 --- a/src/Storages/IPartitionStrategy.cpp +++ b/src/Storages/IPartitionStrategy.cpp @@ -352,97 +352,7 @@ HiveStylePartitionStrategy::HiveStylePartitionStrategy( cacheDeterministicActions(cached_result, actions_with_column); } -<<<<<<< HEAD -ColumnPtr HiveStylePartitionStrategy::computePartitionKey(const Chunk & chunk) const -======= -std::string HiveStylePartitionStrategy::getPathForRead(const std::string & prefix) -{ - /// Match every file extension registered for the format, not only the lowercased format - /// name: for most formats the name is not the extension real files carry (`JSONEachRow` - /// files are named `.jsonl` / `.ndjson`, `CSVWithNames` files are named `.csv`), and a - /// glob built from the format name alone silently matches nothing over such a lake. - /// The lowercased format name is always one of the registered extensions, so the files - /// ClickHouse itself writes (see getPathForWrite) keep matching. - const auto extensions = FormatFactory::instance().getFileExtensionsForFormat(file_format); - - /// The glob is matched against the object key as it is stored, before anything decompresses it - /// (`GlobIterator` filters the listing, while the compression method is derived much later, in - /// `ReadBufferIterator`), so a compressed lake of `key=1/data.jsonl.gz` objects is invisible to - /// a glob of bare extensions. The glob has to accept exactly the names the reader would accept, - /// which is the rule of `chooseCompressionMethod`: - /// - `auto` (the default): the file name decides, so spell out every suffix it recognizes; - /// - an explicit codec: the file name is ignored, so a `gzip` lake of `data.jsonl.custom` objects - /// is as readable as one of `data.jsonl.gz` objects, and anything after the format extension - /// must match. The bare extension is kept too, because an explicit codec also applies to - /// files named without any suffix - and that is what `getPathForWrite` produces; - /// - `none`: the files carry no compression layer, so a suffix after the format extension is - /// not a compression spelling to accept but foreign data (`.parquet.crc` sidecars and the - /// like), and only the bare extensions match. - Strings alternatives = extensions; - - std::string compression_hint = compression_method; - boost::algorithm::to_lower(compression_hint); - - if (compression_hint.empty() || compression_hint == "auto") - { - for (const auto & compression_suffix : getFileSuffixesForCompressionMethodHint(compression_method)) - for (const auto & extension : extensions) - alternatives.push_back(extension + "." + compression_suffix); - } - else if (compression_hint != "none") - { - /// A misspelled codec is not validated here: this runs on `ATTACH` and at server startup too, - /// where throwing would make existing metadata unloadable. `CREATE TABLE` rejects it in - /// `StorageObjectStorageConfiguration::initPartitionStrategy`, and a table attached from older - /// metadata keeps the suffix alternatives, so its reads reach `chooseCompressionMethod` and - /// fail loudly with `Unknown compression method` instead of silently matching nothing. - /// - /// The suffix is arbitrary, but it has to be a suffix: `.csv.*` and not `.csv*`, which is a - /// prefix match on the extension and would hand the `data.csvwithnames.gz` files of a sibling - /// `CSVWithNames` lake to the `CSV` parser (the reader trusts the explicit codec and never looks - /// at the name). The format boundary is the `.` after the extension. - for (const auto & extension : extensions) - alternatives.push_back(extension + ".*"); - } - - if (alternatives.size() == 1) - return prefix + "**." + alternatives.front(); - - return prefix + "**.{" + boost::algorithm::join(alternatives, ",") + "}"; -} - -std::string HiveStylePartitionStrategy::getPathForWrite( - const std::string & prefix, - const std::string & partition_key) -{ - std::string path; - - if (!prefix.empty()) - { - path += prefix; - if (path.back() != '/') - { - path += '/'; - } - } - - /// Not adding '/' because buildExpressionHive() always adds a trailing '/' - path += partition_key; - - /* - * File extension is toLower(format) - * This isn't ideal, but I guess multiple formats can be specified and introduced. - * So I think it is simpler to keep it this way. - * - * Or perhaps implement something like `IInputFormat::getFileExtension()` - */ - path += std::to_string(generateSnowflakeID()) + "." + Poco::toLower(file_format); - - return path; -} - IPartitionStrategy::PartitionExpressionActionsAndColumnName HiveStylePartitionStrategy::getPartitionKeyActions() const ->>>>>>> 690d3be35e1 (Merge pull request #128217 from groeneai/fix-partition-by-stateful-function-spans-chunks) { return getCachedOrBuildActions( cached_result, diff --git a/src/Storages/IPartitionStrategy.h b/src/Storages/IPartitionStrategy.h index 901db76eef5f0..1a905d35243db 100644 --- a/src/Storages/IPartitionStrategy.h +++ b/src/Storages/IPartitionStrategy.h @@ -33,6 +33,9 @@ struct IPartitionStrategy ColumnPtr computePartitionKey(const Chunk & chunk, const PartitionExpressionActionsAndColumnName & partition_key_actions) const; + /// Convenience overload (used by Delta Lake sink): builds the actions on every call. + ColumnPtr computePartitionKey(const Chunk & chunk) const { return computePartitionKey(chunk, getPartitionKeyActions()); } + virtual ColumnPtr computePartitionKey(Block & block) const = 0; virtual ColumnRawPtrs getFormatChunkColumns(const Chunk & chunk) @@ -95,15 +98,9 @@ struct WildcardPartitionStrategy : IPartitionStrategy { WildcardPartitionStrategy(KeyDescription partition_key_description_, const Block & sample_block_, ContextPtr context_); -<<<<<<< HEAD - ColumnPtr computePartitionKey(const Chunk & chunk) const override; + PartitionExpressionActionsAndColumnName getPartitionKeyActions() const override; ColumnPtr computePartitionKey(Block & block) const override; -======= - PartitionExpressionActionsAndColumnName getPartitionKeyActions() const override; - std::string getPathForRead(const std::string & prefix) override; - std::string getPathForWrite(const std::string & prefix, const std::string & partition_key) override; ->>>>>>> 690d3be35e1 (Merge pull request #128217 from groeneai/fix-partition-by-stateful-function-spans-chunks) }; /* @@ -120,15 +117,9 @@ struct HiveStylePartitionStrategy : IPartitionStrategy const std::string & file_format_, bool partition_columns_in_data_file_); -<<<<<<< HEAD - ColumnPtr computePartitionKey(const Chunk & chunk) const override; + PartitionExpressionActionsAndColumnName getPartitionKeyActions() const override; ColumnPtr computePartitionKey(Block & block) const override; -======= - PartitionExpressionActionsAndColumnName getPartitionKeyActions() const override; - std::string getPathForRead(const std::string & prefix) override; - std::string getPathForWrite(const std::string & prefix, const std::string & partition_key) override; ->>>>>>> 690d3be35e1 (Merge pull request #128217 from groeneai/fix-partition-by-stateful-function-spans-chunks) ColumnRawPtrs getFormatChunkColumns(const Chunk & chunk) override; Block getFormatHeader() override;