Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/Interpreters/InterpreterSelectQuery.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2812,7 +2812,7 @@ std::optional<UInt64> InterpreterSelectQuery::getTrivialCount(UInt64 allow_exper
/// require reading some data (but much faster than reading columns).
/// Set a special flag in query info so the storage will see it and optimize count in read() method.
query_info.optimize_trivial_count = optimize_trivial_count;
return storage->totalRows(context);
return storage->totalRowsInSnapshot(storage_snapshot, context);
}

// It's possible to optimize count() given only partition predicates
Expand Down
5 changes: 3 additions & 2 deletions src/Planner/PlannerJoinTree.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -780,8 +780,9 @@ bool applyTrivialCountIfPossible(
/// Set a special flag in query info so the storage will see it and optimize count in read() method.
select_query_info.optimize_trivial_count = true;

/// Get number of rows
std::optional<UInt64> num_rows = storage->totalRows(query_context);
/// Get number of rows in the snapshot this query pinned, so a count does not re-resolve table metadata.
const auto & storage_snapshot = table_node ? table_node->getStorageSnapshot() : table_function_node->getStorageSnapshot();
std::optional<UInt64> num_rows = storage->totalRowsInSnapshot(storage_snapshot, query_context);
if (!num_rows)
return false;

Expand Down
5 changes: 5 additions & 0 deletions src/Storages/IStorage.h
Original file line number Diff line number Diff line change
Expand Up @@ -867,6 +867,11 @@ It is currently only implemented in StorageObjectStorage.
/// Does takes underlying Storage (if any) into account.
virtual std::optional<UInt64> totalRows(ContextPtr) const { return {}; }

virtual std::optional<UInt64> totalRowsInSnapshot(const StorageSnapshotPtr & /*storage_snapshot*/, ContextPtr query_context) const
{
return totalRows(query_context);
}

/// Same as above but also take partition predicate into account.
virtual std::optional<UInt64> totalRowsByPartitionPredicate(const ActionsDAG &, ContextPtr) const { return {}; }

Expand Down
9 changes: 6 additions & 3 deletions src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h
Original file line number Diff line number Diff line change
Expand Up @@ -263,9 +263,9 @@ class DataLakeConfiguration : public BaseStorageConfiguration, public std::enabl
return DataLakeMetadata::supportsTotalRows(context, storage_type);
}

std::optional<size_t> totalRows(ContextPtr local_context) override
std::optional<size_t> totalRows(ContextPtr local_context, const std::optional<DataLakeTableStateSnapshot> & state) override
{
return getMetadata()->totalRows(local_context);
return getMetadata()->totalRows(local_context, state);
}

bool supportsTotalBytes(ContextPtr context, ObjectStorageType storage_type) const override
Expand Down Expand Up @@ -661,7 +661,10 @@ class StorageIcebergConfiguration : public StorageObjectStorageConfiguration, pu
bool isDataLakeConfiguration() const override { return getImpl().isDataLakeConfiguration(); }

bool supportsTotalRows(ContextPtr context, ObjectStorageType storage_type) const override { return getImpl().supportsTotalRows(context, storage_type); }
std::optional<size_t> totalRows(ContextPtr context) override { return getImpl().totalRows(context); }
std::optional<size_t> totalRows(ContextPtr context, const std::optional<DataLakeTableStateSnapshot> & state) override
{
return getImpl().totalRows(context, state);
}
bool supportsTotalBytes(ContextPtr context, ObjectStorageType storage_type) const override { return getImpl().supportsTotalBytes(context, storage_type); }
std::optional<size_t> totalBytes(ContextPtr context) override { return getImpl().totalBytes(context); }
bool isDataSortedBySortingKey(StorageMetadataPtr storage_metadata, ContextPtr context) const override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ bool DeltaLakeMetadataDeltaKernel::operator ==(const IDataLakeMetadata & metadat
return getTableSnapshot()->getVersion() == delta_lake_metadata.getTableSnapshot()->getVersion();
}

std::optional<size_t> DeltaLakeMetadataDeltaKernel::totalRows(ContextPtr context) const
std::optional<size_t> DeltaLakeMetadataDeltaKernel::totalRows(ContextPtr context, const std::optional<DataLakeTableStateSnapshot> &) const
{
const auto & settings = context->getSettingsRef();
if (auto start_version = settings[Setting::delta_lake_snapshot_start_version].value;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ class DeltaLakeMetadataDeltaKernel final : public IDataLakeMetadata
return std::make_unique<DeltaLakeMetadataDeltaKernel>(object_storage_, configuration);
}

std::optional<size_t> totalRows(ContextPtr) const override;
std::optional<size_t> totalRows(ContextPtr, const std::optional<DataLakeTableStateSnapshot> &) const override;

std::optional<size_t> totalBytes(ContextPtr) const override;

Expand Down
2 changes: 1 addition & 1 deletion src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ class IDataLakeMetadata : boost::noncopyable
virtual void modifyFormatSettings(FormatSettings &, const Context &) const {}

static bool supportsTotalRows(ContextPtr, ObjectStorageType) { return false; }
virtual std::optional<size_t> totalRows(ContextPtr) const { return {}; }
virtual std::optional<size_t> totalRows(ContextPtr, const std::optional<DataLakeTableStateSnapshot> &) const { return {}; }
static bool supportsTotalBytes(ContextPtr, ObjectStorageType) { return false; }
virtual std::optional<size_t> totalBytes(ContextPtr) const { return {}; }

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -429,14 +429,14 @@ std::optional<AlterDropPartitionExecutor::SnapshotState> AlterDropPartitionExecu
}

auto compression_method = DB::Iceberg::getCompressionMethodFromMetadataFile(state.table_state.metadata_file_path);
auto metadata_object = getMetadataJSONObject(
auto metadata_object = deepCopy(getMetadataJSONObject(
state.table_state.metadata_file_path,
object_storage,
components.metadata_cache,
context,
log,
compression_method,
components.table_uuid);
components.table_uuid));

const auto format_version = metadata_object->getValue<Int32>(f_format_version);
if (format_version != 2)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -649,14 +649,14 @@ LatestMetadata readLatestMetadata(
/* force_fetch_latest_metadata */ true,
/* ignore_explicit_metadata_file_path */ false);

auto metadata_object = getMetadataJSONObject(
auto metadata_object = deepCopy(getMetadataJSONObject(
catalog_metadata.path,
object_storage,
persistent_table_components.metadata_cache,
context,
log,
catalog_metadata.compression_method,
persistent_table_components.table_uuid);
persistent_table_components.table_uuid));

/// The new metadata file must not reuse the name of a newer file left in the directory.
const auto max_directory_version = getMaxMetadataFileVersionInDirectory(object_storage, persistent_table_components.table_path);
Expand All @@ -677,14 +677,14 @@ LatestMetadata readLatestMetadata(
/* force_fetch_latest_metadata */ true,
/* ignore_explicit_metadata_file_path */ true);

auto metadata_object = getMetadataJSONObject(
auto metadata_object = deepCopy(getMetadataJSONObject(
metadata_file_path,
object_storage,
persistent_table_components.metadata_cache,
context,
log,
persistent_table_components.metadata_compression_method,
persistent_table_components.table_uuid);
persistent_table_components.table_uuid));

return {metadata_version, metadata_object, {}};
}
Expand Down
6 changes: 3 additions & 3 deletions src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ static Plan getPlan(
persistent_table_components.metadata_compression_method);

Poco::JSON::Object::Ptr initial_metadata_object
= getMetadataJSONObject(metadata_file_path, object_storage, persistent_table_components.metadata_cache, context, log, compression_method, persistent_table_components.table_uuid);
= deepCopy(getMetadataJSONObject(metadata_file_path, object_storage, persistent_table_components.metadata_cache, context, log, compression_method, persistent_table_components.table_uuid));

/// Exactly version 2: v1 lacks the sequence-number machinery the rewrite relies on, and
/// a v3 table must not be accepted either -- writeMetadataFiles rebuilds the metadata
Expand Down Expand Up @@ -1548,14 +1548,14 @@ void compactIcebergManifests(
/* force_fetch_latest_metadata */ true,
/* ignore_explicit_metadata_file_path */ true);

auto metadata_object = getMetadataJSONObject(
auto metadata_object = deepCopy(getMetadataJSONObject(
metadata_file_path,
object_storage_,
persistent_table_components.metadata_cache,
context_,
log,
persistent_table_components.metadata_compression_method,
persistent_table_components.table_uuid);
persistent_table_components.table_uuid));

/// Validate the format version on the freshly-fetched metadata (before the threshold early-return), since the table may have been upgraded to v3 by another writer after this table object was created.
const Int32 format_version = metadata_object->getValue<Int32>(Iceberg::f_format_version);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -856,14 +856,14 @@ ExpireSnapshotsResult expireSnapshots(
filename_generator.setVersion(last_version + 1);
filename_generator.setCompressionMethod(compression_method);

auto metadata = getMetadataJSONObject(
auto metadata = deepCopy(getMetadataJSONObject(
metadata_path,
object_storage,
persistent_table_components.metadata_cache,
context,
log,
compression_method,
persistent_table_components.table_uuid);
persistent_table_components.table_uuid));

if (metadata->getValue<Int32>(f_format_version) < 2)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "expire_snapshots is supported only for the second version of iceberg format");
Expand Down
Loading
Loading