Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 14 additions & 16 deletions src/Storages/IPartitionStrategy.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,16 @@ IPartitionStrategy::getPartitionExpressionActions(ASTPtr & expression_ast) const
return result;
}

ColumnPtr IPartitionStrategy::computePartitionKey(
const Chunk & chunk, const PartitionExpressionActionsAndColumnName & partition_key_actions) const
{
Block block_with_partition_by_expr = sample_block.cloneWithoutColumns();
block_with_partition_by_expr.setColumns(chunk.getColumns());
partition_key_actions.actions->execute(block_with_partition_by_expr);

return block_with_partition_by_expr.getByName(partition_key_actions.column_name).column;
}

std::shared_ptr<IPartitionStrategy> PartitionStrategyFactory::get(StrategyType strategy,
ASTPtr partition_by,
const NamesAndTypesList & partition_columns,
Expand Down Expand Up @@ -298,18 +308,12 @@ WildcardPartitionStrategy::WildcardPartitionStrategy(KeyDescription partition_ke
cacheDeterministicActions(cached_result, actions_with_column);
}

ColumnPtr WildcardPartitionStrategy::computePartitionKey(const Chunk & chunk) const
IPartitionStrategy::PartitionExpressionActionsAndColumnName WildcardPartitionStrategy::getPartitionKeyActions() const
{
auto actions_with_column = getCachedOrBuildActions(
return getCachedOrBuildActions(
cached_result,
*this,
[&] { return buildToStringPartitionAST(partition_key_description.definition_ast); });

Block block_with_partition_by_expr = sample_block.cloneWithoutColumns();
block_with_partition_by_expr.setColumns(chunk.getColumns());
actions_with_column.actions->execute(block_with_partition_by_expr);

return block_with_partition_by_expr.getByName(actions_with_column.column_name).column;
}

ColumnPtr WildcardPartitionStrategy::computePartitionKey(Block & block) const
Expand Down Expand Up @@ -348,18 +352,12 @@ HiveStylePartitionStrategy::HiveStylePartitionStrategy(
cacheDeterministicActions(cached_result, actions_with_column);
}

ColumnPtr HiveStylePartitionStrategy::computePartitionKey(const Chunk & chunk) const
IPartitionStrategy::PartitionExpressionActionsAndColumnName HiveStylePartitionStrategy::getPartitionKeyActions() const
{
auto actions_with_column = getCachedOrBuildActions(
return getCachedOrBuildActions(
cached_result,
*this,
[&] { return buildHivePartitionAST(partition_key_description.definition_ast, getPartitionColumns()); });

Block block_with_partition_by_expr = sample_block.cloneWithoutColumns();
block_with_partition_by_expr.setColumns(chunk.getColumns());
actions_with_column.actions->execute(block_with_partition_by_expr);

return block_with_partition_by_expr.getByName(actions_with_column.column_name).column;
}

ColumnPtr HiveStylePartitionStrategy::computePartitionKey(Block & block) const
Expand Down
13 changes: 10 additions & 3 deletions src/Storages/IPartitionStrategy.h
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,14 @@ struct IPartitionStrategy

virtual ~IPartitionStrategy() = default;

virtual ColumnPtr computePartitionKey(const Chunk & chunk) const = 0;
/// Rebuilt on every call if non-deterministic. A sink calls it once and reuses the result for all its chunks,
/// so a stateful function keeps its state across the chunks of one INSERT.
virtual PartitionExpressionActionsAndColumnName getPartitionKeyActions() const = 0;

ColumnPtr computePartitionKey(const Chunk & chunk, const PartitionExpressionActionsAndColumnName & partition_key_actions) const;

/// Convenience overload (used by Delta Lake sink): builds the actions on every call.
ColumnPtr computePartitionKey(const Chunk & chunk) const { return computePartitionKey(chunk, getPartitionKeyActions()); }

virtual ColumnPtr computePartitionKey(Block & block) const = 0;

Expand Down Expand Up @@ -91,7 +98,7 @@ struct WildcardPartitionStrategy : IPartitionStrategy
{
WildcardPartitionStrategy(KeyDescription partition_key_description_, const Block & sample_block_, ContextPtr context_);

ColumnPtr computePartitionKey(const Chunk & chunk) const override;
PartitionExpressionActionsAndColumnName getPartitionKeyActions() const override;

ColumnPtr computePartitionKey(Block & block) const override;
};
Expand All @@ -110,7 +117,7 @@ struct HiveStylePartitionStrategy : IPartitionStrategy
const std::string & file_format_,
bool partition_columns_in_data_file_);

ColumnPtr computePartitionKey(const Chunk & chunk) const override;
PartitionExpressionActionsAndColumnName getPartitionKeyActions() const override;

ColumnPtr computePartitionKey(Block & block) const override;

Expand Down
3 changes: 2 additions & 1 deletion src/Storages/PartitionedSink.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ PartitionedSink::PartitionedSink(
, sink_creator(sink_creator_)
, context(context_)
, source_header(source_header_)
, partition_key_actions(partition_strategy->getPartitionKeyActions())
{
}

Expand All @@ -52,7 +53,7 @@ SinkPtr PartitionedSink::getSinkForPartitionKey(std::string_view partition_key)

void PartitionedSink::consume(Chunk & source_chunk)
{
const ColumnPtr partition_by_result_column = partition_strategy->computePartitionKey(source_chunk);
const ColumnPtr partition_by_result_column = partition_strategy->computePartitionKey(source_chunk, partition_key_actions);

/// Not all columns are serialized using the format writer (e.g, hive partitioning stores partition columns in the file path)
const auto columns_to_consume = partition_strategy->getFormatChunkColumns(source_chunk);
Expand Down
1 change: 1 addition & 0 deletions src/Storages/PartitionedSink.h
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ class PartitionedSink : public SinkToStorage
std::shared_ptr<SinkCreator> sink_creator;
ContextPtr context;
SharedHeader source_header;
IPartitionStrategy::PartitionExpressionActionsAndColumnName partition_key_actions;

absl::flat_hash_map<std::string_view, SinkPtr> partition_id_to_sink;
HashMapWithSavedHash<std::string_view, size_t> partition_id_to_chunk_index;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
05459_rn_0.csv 1000 0 999
05459_rn_1.csv 1000 1000 1999
05459_rn_2.csv 1000 2000 2999
05459_rn_3.csv 1000 3000 3999
05459_rn_4.csv 1000 4000 4999
05459_eng_0.csv 1000 10000 10999
05459_eng_1.csv 1000 11000 11999
05459_eng_2.csv 1000 2000 2999
05459_eng_3.csv 1000 3000 3999
05459_eng_4.csv 1000 4000 4999
1 5000
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
-- Tags: no-fasttest
-- Tag no-fasttest: uses S3

SET max_block_size = 1000, max_insert_threads = 1, engine_file_truncate_on_insert = 1;

-- file(): the counter spans the five 1000-row chunks
INSERT INTO FUNCTION file(currentDatabase() || '/05459_rn_{_partition_id}.csv', 'CSV', 'n UInt64')
PARTITION BY intDiv(rowNumberInAllBlocks(), 1000)
SELECT number FROM numbers(5000);
SELECT _file, count(), min(n), max(n) FROM file(currentDatabase() || '/05459_rn_*.csv', 'CSV', 'n UInt64') GROUP BY _file ORDER BY _file;

-- S3 engine (strategy shared by all INSERTs into the table): spans chunks, starts anew with the next INSERT
CREATE TABLE t_05459 (n UInt64)
ENGINE = S3(s3_conn, filename = currentDatabase() || '/05459_eng_{_partition_id}.csv', format = CSV, partition_strategy = 'wildcard')
PARTITION BY intDiv(rowNumberInAllBlocks(), 1000);
INSERT INTO t_05459 SELECT number FROM numbers(5000) SETTINGS s3_truncate_on_insert = 1, s3_create_new_file_on_insert = 0;
INSERT INTO t_05459 SELECT number + 10000 FROM numbers(2000) SETTINGS s3_truncate_on_insert = 1, s3_create_new_file_on_insert = 0;
SELECT _file, count(), min(n), max(n) FROM s3(s3_conn, filename = currentDatabase() || '/05459_eng_*.csv', format = CSV, structure = 'n UInt64') GROUP BY _file ORDER BY _file;
DROP TABLE t_05459;

-- now64() is evaluated once per INSERT, not once per chunk
INSERT INTO FUNCTION file(currentDatabase() || '/05459_now_{_partition_id}.csv', 'CSV', 'n UInt64')
PARTITION BY toUnixTimestamp64Nano(now64(9))
SELECT number FROM numbers(5000);
SELECT count(DISTINCT _file), count() FROM file(currentDatabase() || '/05459_now_*.csv', 'CSV', 'n UInt64');
Loading