From c873aa2ef8ac8ec87287458821ebac19f07c5945 Mon Sep 17 00:00:00 2001 From: Konstantin Vedernikov <75157521+scanhex12@users.noreply.github.com> Date: Thu, 22 Oct 2026 13:14:13 +0000 Subject: [PATCH 1/2] Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json Use less s3 requests to get metadata file Conflicts committed as-is; the next commit resolves them: - src/Planner/PlannerJoinTree.cpp (both modified) - src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadataDeltaKernel.h (both modified) - src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp (both modified) - src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h (both modified) - src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadataFilesCache.h (both modified) - src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp (both modified) - src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp (both modified) - src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp (both modified) - src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp (both modified) - src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp (both modified) --- src/Interpreters/InterpreterSelectQuery.cpp | 2 +- src/Planner/PlannerJoinTree.cpp | 20 +++ src/Storages/IStorage.h | 5 + .../DataLakes/DataLakeConfiguration.h | 4 +- .../DeltaLakeMetadataDeltaKernel.cpp | 2 +- .../DataLakes/DeltaLakeMetadataDeltaKernel.h | 17 +++ .../DataLakes/IDataLakeMetadata.h | 2 +- .../Iceberg/AlterDropPartitionExecutor.cpp | 4 +- .../DataLakes/Iceberg/Compaction.cpp | 6 +- .../Iceberg/ExpireSnapshotsExecute.cpp | 4 +- .../DataLakes/Iceberg/IcebergMetadata.cpp | 116 ++++++++++++++++-- .../DataLakes/Iceberg/IcebergMetadata.h | 26 +++- .../Iceberg/IcebergMetadataFilesCache.h | 27 ++-- .../DataLakes/Iceberg/IcebergWrites.cpp | 10 +- .../DataLakes/Iceberg/MetadataGenerator.cpp | 82 +++++++++++++ .../DataLakes/Iceberg/Mutations.cpp | 6 +- .../ObjectStorage/DataLakes/Iceberg/Utils.cpp | 105 +++++++++++++++- .../ObjectStorage/DataLakes/Iceberg/Utils.h | 12 ++ .../ObjectStorage/StorageObjectStorage.cpp | 16 ++- .../ObjectStorage/StorageObjectStorage.h | 1 + .../StorageObjectStorageCluster.cpp | 15 ++- .../StorageObjectStorageCluster.h | 1 + .../StorageObjectStorageConfiguration.h | 2 +- ...407_iceberg_metadata_s3_requests.reference | 4 + .../05407_iceberg_metadata_s3_requests.sh | 33 +++++ ...cted_alter_keeps_cached_metadata.reference | 4 + ...rg_rejected_alter_keeps_cached_metadata.sh | 24 ++++ ..._sees_rows_appended_after_create.reference | 1 + ..._select_sees_rows_appended_after_create.sh | 17 +++ ...on_lower_version_replaces_cached.reference | 2 + ...selection_lower_version_replaces_cached.sh | 29 +++++ 31 files changed, 552 insertions(+), 47 deletions(-) create mode 100644 tests/queries/0_stateless/05407_iceberg_metadata_s3_requests.reference create mode 100755 tests/queries/0_stateless/05407_iceberg_metadata_s3_requests.sh create mode 100644 tests/queries/0_stateless/05408_iceberg_rejected_alter_keeps_cached_metadata.reference create mode 100755 tests/queries/0_stateless/05408_iceberg_rejected_alter_keeps_cached_metadata.sh create mode 100644 tests/queries/0_stateless/05409_iceberg_select_sees_rows_appended_after_create.reference create mode 100755 tests/queries/0_stateless/05409_iceberg_select_sees_rows_appended_after_create.sh create mode 100644 tests/queries/0_stateless/05441_iceberg_timestamp_selection_lower_version_replaces_cached.reference create mode 100755 tests/queries/0_stateless/05441_iceberg_timestamp_selection_lower_version_replaces_cached.sh diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index f7ab3caf4e851..e89196b605251 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -2812,7 +2812,7 @@ std::optional InterpreterSelectQuery::getTrivialCount(UInt64 allow_exper /// require reading some data (but much faster than reading columns). /// Set a special flag in query info so the storage will see it and optimize count in read() method. query_info.optimize_trivial_count = optimize_trivial_count; - return storage->totalRows(context); + return storage->totalRowsInSnapshot(storage_snapshot, context); } // It's possible to optimize count() given only partition predicates diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index e5cf42e3d566e..e247d321e9b55 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -780,8 +780,28 @@ bool applyTrivialCountIfPossible( /// Set a special flag in query info so the storage will see it and optimize count in read() method. select_query_info.optimize_trivial_count = true; +<<<<<<< HEAD /// Get number of rows std::optional num_rows = storage->totalRows(query_context); +======= + /// Do not apply trivial count optimization if we're the follower. Because if every follower blindly returns totalRows(), + /// the result will be R times the actual count of rows, where R is the number of replicas. + /// It's important to make this check after we've set the `select_query_info.optimize_trivial_count` flag, so that storage could apply + /// some optimization during the read phase. + /// TODO: We could still avoid reading data for MergeTree on the followers: we could read the parts metadata for the set of parts + /// we were handed out by the initiator. This would require consuming the `select_query_info.optimize_trivial_count` during the read + /// phase and acting accordingly. + if (query_context->canUseParallelReplicasOnFollower()) + return false; + + /// Get number of rows. + /// MergeTree maintains total count for all parts in Active state and it simply returns that number for trivial + /// `select count() from table` query. But if we have current transaction, then we should return number of rows in current snapshot + /// (that may include parts in Outdated state), so the totalRows for transactional case should return the number of rows visible for + /// the current transaction. + const auto & storage_snapshot = table_node ? table_node->getStorageSnapshot() : table_function_node->getStorageSnapshot(); + std::optional num_rows = storage->totalRowsInSnapshot(storage_snapshot, query_context); +>>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) if (!num_rows) return false; diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 95696fdb27b5f..2e91d83f91886 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -867,6 +867,11 @@ It is currently only implemented in StorageObjectStorage. /// Does takes underlying Storage (if any) into account. virtual std::optional totalRows(ContextPtr) const { return {}; } + virtual std::optional totalRowsInSnapshot(const StorageSnapshotPtr & /*storage_snapshot*/, ContextPtr query_context) const + { + return totalRows(query_context); + } + /// Same as above but also take partition predicate into account. virtual std::optional totalRowsByPartitionPredicate(const ActionsDAG &, ContextPtr) const { return {}; } diff --git a/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h b/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h index c5f5865f964dd..2a23cea3dac75 100644 --- a/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h +++ b/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h @@ -263,9 +263,9 @@ class DataLakeConfiguration : public BaseStorageConfiguration, public std::enabl return DataLakeMetadata::supportsTotalRows(context, storage_type); } - std::optional totalRows(ContextPtr local_context) override + std::optional totalRows(ContextPtr local_context, const std::optional & state) override { - return getMetadata()->totalRows(local_context); + return getMetadata()->totalRows(local_context, state); } bool supportsTotalBytes(ContextPtr context, ObjectStorageType storage_type) const override diff --git a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadataDeltaKernel.cpp b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadataDeltaKernel.cpp index 359110153e2b3..0120b326aa74f 100644 --- a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadataDeltaKernel.cpp +++ b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadataDeltaKernel.cpp @@ -202,7 +202,7 @@ bool DeltaLakeMetadataDeltaKernel::operator ==(const IDataLakeMetadata & metadat return getTableSnapshot()->getVersion() == delta_lake_metadata.getTableSnapshot()->getVersion(); } -std::optional DeltaLakeMetadataDeltaKernel::totalRows(ContextPtr context) const +std::optional DeltaLakeMetadataDeltaKernel::totalRows(ContextPtr context, const std::optional &) const { const auto & settings = context->getSettingsRef(); if (auto start_version = settings[Setting::delta_lake_snapshot_start_version].value; diff --git a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadataDeltaKernel.h b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadataDeltaKernel.h index 50c10de476bf3..214b5ceb38a1a 100644 --- a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadataDeltaKernel.h +++ b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadataDeltaKernel.h @@ -69,7 +69,24 @@ class DeltaLakeMetadataDeltaKernel final : public IDataLakeMetadata return std::make_unique(object_storage_, configuration); } +<<<<<<< HEAD std::optional totalRows(ContextPtr) const override; +======= + /// CREATE TABLE entry point when the kernel is enabled: validate the declared columns, write commit 0 + /// (fresh) or attach to an existing `_delta_log`, and register the table in the catalog when present. + static void createInitial( + const ObjectStoragePtr & object_storage, + const StorageObjectStorageConfigurationWeakPtr & configuration, + const ContextPtr & local_context, + const std::optional & columns, + ASTPtr partition_by, + ASTPtr order_by, + bool if_not_exists, + std::shared_ptr catalog, + const StorageID & table_id_); + + std::optional totalRows(ContextPtr, const std::optional &) const override; +>>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) std::optional totalBytes(ContextPtr) const override; diff --git a/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h b/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h index c6add3a39a142..f7947ff37ca08 100644 --- a/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h @@ -183,7 +183,7 @@ class IDataLakeMetadata : boost::noncopyable virtual void modifyFormatSettings(FormatSettings &, const Context &) const {} static bool supportsTotalRows(ContextPtr, ObjectStorageType) { return false; } - virtual std::optional totalRows(ContextPtr) const { return {}; } + virtual std::optional totalRows(ContextPtr, const std::optional &) const { return {}; } static bool supportsTotalBytes(ContextPtr, ObjectStorageType) { return false; } virtual std::optional totalBytes(ContextPtr) const { return {}; } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.cpp index f9e4db801455a..54812df2effcf 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.cpp @@ -429,14 +429,14 @@ std::optional AlterDropPartitionExecu } auto compression_method = DB::Iceberg::getCompressionMethodFromMetadataFile(state.table_state.metadata_file_path); - auto metadata_object = getMetadataJSONObject( + auto metadata_object = deepCopy(getMetadataJSONObject( state.table_state.metadata_file_path, object_storage, components.metadata_cache, context, log, compression_method, - components.table_uuid); + components.table_uuid)); const auto format_version = metadata_object->getValue(f_format_version); if (format_version != 2) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp index 941b9d1a69f5f..24cdfe3e7cbcb 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp @@ -217,7 +217,7 @@ static Plan getPlan( persistent_table_components.metadata_compression_method); Poco::JSON::Object::Ptr initial_metadata_object - = getMetadataJSONObject(metadata_file_path, object_storage, persistent_table_components.metadata_cache, context, log, compression_method, persistent_table_components.table_uuid); + = deepCopy(getMetadataJSONObject(metadata_file_path, object_storage, persistent_table_components.metadata_cache, context, log, compression_method, persistent_table_components.table_uuid)); /// Exactly version 2: v1 lacks the sequence-number machinery the rewrite relies on, and /// a v3 table must not be accepted either -- writeMetadataFiles rebuilds the metadata @@ -1548,14 +1548,14 @@ void compactIcebergManifests( /* force_fetch_latest_metadata */ true, /* ignore_explicit_metadata_file_path */ true); - auto metadata_object = getMetadataJSONObject( + auto metadata_object = deepCopy(getMetadataJSONObject( metadata_file_path, object_storage_, persistent_table_components.metadata_cache, context_, log, persistent_table_components.metadata_compression_method, - persistent_table_components.table_uuid); + persistent_table_components.table_uuid)); /// Validate the format version on the freshly-fetched metadata (before the threshold early-return), since the table may have been upgraded to v3 by another writer after this table object was created. const Int32 format_version = metadata_object->getValue(Iceberg::f_format_version); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ExpireSnapshotsExecute.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ExpireSnapshotsExecute.cpp index 4d1418fd30ad0..69cc6ce8d14df 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ExpireSnapshotsExecute.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ExpireSnapshotsExecute.cpp @@ -856,14 +856,14 @@ ExpireSnapshotsResult expireSnapshots( filename_generator.setVersion(last_version + 1); filename_generator.setCompressionMethod(compression_method); - auto metadata = getMetadataJSONObject( + auto metadata = deepCopy(getMetadataJSONObject( metadata_path, object_storage, persistent_table_components.metadata_cache, context, log, compression_method, - persistent_table_components.table_uuid); + persistent_table_components.table_uuid)); if (metadata->getValue(f_format_version) < 2) throw Exception(ErrorCodes::BAD_ARGUMENTS, "expire_snapshots is supported only for the second version of iceberg format"); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index 561df9a7504c4..269287fe046a7 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -243,18 +243,19 @@ Iceberg::TableStateSnapshotPtr extractIcebergSnapshotIdFromMetadataObject(Storag } -Iceberg::PersistentTableComponents IcebergMetadata::initializePersistentTableComponents( +std::pair IcebergMetadata::initializePersistentTableComponents( ObjectStoragePtr object_storage, StorageObjectStorageConfigurationPtr configuration, IcebergMetadataFilesCachePtr cache_ptr, ContextPtr context_, LoggerPtr log) { - const auto [metadata_version, metadata_file_path, compression_method] + const auto latest_metadata = getLatestOrExplicitMetadataFileAndVersion(object_storage, configuration->getPathForRead().path, configuration->getDataLakeSettings(), cache_ptr, context_, log.get(), std::nullopt, CompressionMethod::None, /* force_fetch_latest_metadata */ false); + const auto & [metadata_version, metadata_file_path, compression_method] = latest_metadata; LOG_DEBUG(log, "Latest metadata file path is {}, version {}", metadata_file_path, metadata_version); - auto metadata_object - = getMetadataJSONObject(metadata_file_path, object_storage, cache_ptr, context_, log, compression_method, std::nullopt); + auto metadata_json_str = readMetadataJSONString(metadata_file_path, object_storage, cache_ptr != nullptr, context_, log, compression_method); + auto metadata_object = parseMetadataJSONString(metadata_json_str); Int32 format_version = metadata_object->getValue(f_format_version); String table_location = metadata_object->getValue(f_location); std::optional table_uuid = std::nullopt; @@ -273,11 +274,20 @@ Iceberg::PersistentTableComponents IcebergMetadata::initializePersistentTableCom Iceberg::f_table_uuid); } } + if (cache_ptr && table_uuid) + cache_ptr->getOrSetTableMetadata( + IcebergMetadataFilesCache::getKey(*table_uuid, metadata_file_path), + [&]() { return std::make_shared(metadata_object, metadata_json_str.size()); }); auto table_path = configuration->getPathForRead().path; auto root_derivation = IcebergPathResolver::deriveTableRoot(table_location, table_path, metadata_file_path); +<<<<<<< HEAD return PersistentTableComponents{ .schema_processor = std::make_shared( context_, context_->getSettingsRef()[Setting::allow_experimental_geo_types_in_iceberg]), +======= + PersistentTableComponents persistent_components{ + .schema_processor = std::make_shared(context_->getSettingsRef()[Setting::allow_geo_types_in_iceberg]), +>>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) .metadata_cache = cache_ptr, .format_version = format_version, .table_location = table_location, @@ -289,13 +299,80 @@ Iceberg::PersistentTableComponents IcebergMetadata::initializePersistentTableCom .table_root_was_derived = root_derivation.relation == IcebergPathResolver::RootRelation::AdoptedDescendant, .common_namespace = configuration->getNamespace(), }; + return {std::move(persistent_components), latest_metadata}; +} + +<<<<<<< HEAD +======= +void IcebergMetadata::setExplicitMetadataFilePath(const String & path) +{ + explicit_metadata_file_path.set(std::make_unique(path)); +} + +void IcebergMetadata::update(const ContextPtr &) +{ + std::lock_guard lock(cached_latest_metadata_mutex); + cached_latest_metadata.reset(); +} + +Iceberg::MetadataFileWithInfo IcebergMetadata::fetchLatestMetadata( + const ContextPtr & context, bool force_fetch_latest_metadata, bool ignore_metadata_pointer_overrides) const +{ + return getLatestOrExplicitMetadataFileAndVersion( + object_storage, + persistent_components.table_path, + getMetadataLookupSettings(), + persistent_components.metadata_cache, + context, + log.get(), + persistent_components.table_uuid, + persistent_components.metadata_compression_method, + force_fetch_latest_metadata, + ignore_metadata_pointer_overrides); +} + +DataLakeStorageSettings IcebergMetadata::getMetadataLookupSettings() const +{ + DataLakeStorageSettings result = data_lake_settings; + if (auto path = explicit_metadata_file_path.get()) + result[DataLakeStorageSetting::iceberg_metadata_file_path] = *path; + return result; } +>>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) std::pair IcebergMetadata::getRelevantState( const ContextPtr & context, bool force_fetch_latest_metadata) const { +<<<<<<< HEAD const auto [metadata_version, metadata_file_path, compression_method] = getLatestOrExplicitMetadataFileAndVersion( object_storage, +======= + ProfileEventTimeIncrement watch(ProfileEvents::IcebergGetTableStateMicroseconds); + if (!force_fetch_latest_metadata && !ignore_metadata_pointer_overrides && !explicit_metadata_file_path.get() + && !data_lake_settings[DataLakeStorageSetting::iceberg_metadata_file_path].changed) + { + std::optional latest_metadata; + { + std::lock_guard lock(cached_latest_metadata_mutex); + latest_metadata = cached_latest_metadata; + } + if (latest_metadata) + return getState(context, latest_metadata->path, latest_metadata->version); + } + + const auto latest_metadata = fetchLatestMetadata(context, force_fetch_latest_metadata, ignore_metadata_pointer_overrides); + return getState(context, latest_metadata.path, latest_metadata.version); +} + +std::pair IcebergMetadata::getRelevantState( + const ContextPtr & context, const std::shared_ptr & catalog, const String & table_identifier) const +{ + ProfileEventTimeIncrement watch(ProfileEvents::IcebergGetTableStateMicroseconds); + const auto [metadata_version, metadata_file_path, _] = getLatestMetadataFileAndVersionWithCatalog( + object_storage, + catalog, + table_identifier, +>>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) persistent_components.table_path, data_lake_settings, persistent_components.metadata_cache, @@ -311,13 +388,19 @@ IcebergMetadata::IcebergMetadata( ObjectStoragePtr object_storage_, StorageObjectStorageConfigurationPtr configuration_, Iceberg::PersistentTableComponents && persistent_components_, - ContextPtr context_) + ContextPtr context_, + std::optional cached_latest_metadata_) : log(getLogger("IcebergMetadata")) , object_storage(std::move(object_storage_)) , external_storages(std::make_shared()) , persistent_components(std::move(persistent_components_)) , data_lake_settings(configuration_->getDataLakeSettings()) +<<<<<<< HEAD , write_format(configuration_->getFormat()) +======= + , write_format(configuration_->format) + , cached_latest_metadata(std::move(cached_latest_metadata_)) +>>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) { /// TODO: for now it's okay to start/stop the task via constructor/destructor. Once refactored, we'd need to plumb startup/shutdown and schedule the task from there if (persistent_components.metadata_cache && data_lake_settings[DataLakeStorageSetting::iceberg_metadata_async_prefetch_period_ms] != 0) @@ -1334,8 +1417,8 @@ DataLakeMetadataPtr IcebergMetadata::create( else LOG_TRACE( log, "Not using in-memory cache for iceberg metadata files, because the setting use_iceberg_metadata_files_cache is false."); - auto persistent_components = initializePersistentTableComponents(object_storage, configuration_ptr, cache_ptr, local_context, log); - return std::make_unique(object_storage, configuration_ptr, std::move(persistent_components), local_context); + auto [persistent_components, latest_metadata] = initializePersistentTableComponents(object_storage, configuration_ptr, cache_ptr, local_context, log); + return std::make_unique(object_storage, configuration_ptr, std::move(persistent_components), local_context, latest_metadata); } IcebergMetadata::IcebergHistory IcebergMetadata::getHistory(ContextPtr local_context) const @@ -1604,9 +1687,17 @@ bool IcebergMetadata::supportsLazyMaterialization(StorageMetadataPtr storage_met return true; } -std::optional IcebergMetadata::totalRows(ContextPtr local_context) const +std::optional IcebergMetadata::totalRows(ContextPtr local_context, const std::optional & state) const { - auto [actual_data_snapshot, actual_table_state_snapshot] = getRelevantState(local_context); + IcebergDataSnapshotPtr actual_data_snapshot; + TableStateSnapshot actual_table_state_snapshot; + if (state) + { + actual_table_state_snapshot = std::get(*state); + actual_data_snapshot = getRelevantDataSnapshotFromTableStateSnapshot(actual_table_state_snapshot, local_context); + } + else + std::tie(actual_data_snapshot, actual_table_state_snapshot) = getRelevantState(local_context); if (!actual_data_snapshot) { @@ -2174,7 +2265,7 @@ DataLakeMetadataPtr IcebergMetadata::createWithDeserialization( auto configuration_ptr = configuration.lock(); if (!configuration_ptr) throw Exception(ErrorCodes::LOGICAL_ERROR, "Trying to create Iceberg table, but storage configuration is expired"); - auto standard_persistent_components + auto [standard_persistent_components, latest_metadata] = initializePersistentTableComponents(object_storage, configuration_ptr, cache_ptr, local_context, log); auto schema_processor = standard_persistent_components.schema_processor; auto deserialized_persistent_components = Iceberg::PersistentTableComponents{ @@ -2191,9 +2282,14 @@ DataLakeMetadataPtr IcebergMetadata::createWithDeserialization( configuration_ptr->getTypeName(), configuration_ptr->getNamespace()), /// Consistent with the resolver above, which is rooted at `table_path` itself. +<<<<<<< HEAD .table_root_was_derived = false, .common_namespace = configuration_ptr->getNamespace()}; auto metadata = std::make_unique(object_storage, configuration.lock(), std::move(deserialized_persistent_components), local_context); +======= + .table_root_was_derived = false}; + auto metadata = std::make_unique(object_storage, configuration.lock(), std::move(deserialized_persistent_components), local_context, latest_metadata); +>>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) return metadata; } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h index 8b95f2eb26e5c..40b9bf4cd01bd 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h @@ -74,7 +74,8 @@ class IcebergMetadata : public IDataLakeMetadata ObjectStoragePtr object_storage_, StorageObjectStorageConfigurationPtr configuration_, Iceberg::PersistentTableComponents && persistent_components_, - ContextPtr context_); + ContextPtr context_, + std::optional cached_latest_metadata_ = std::nullopt); ~IcebergMetadata() override; @@ -113,6 +114,14 @@ class IcebergMetadata : public IDataLakeMetadata LoggerPtr metadata_logger); bool supportsUpdate() const override { return true; } +<<<<<<< HEAD +======= + + void update(const ContextPtr & context) override; + + void setExplicitMetadataFilePath(const String & path) override; + +>>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) bool supportsWrites() const override { return true; } bool supportsParallelInsert() const override { return true; } bool supportsTruncate() const override { return true; } @@ -128,7 +137,7 @@ class IcebergMetadata : public IDataLakeMetadata ContextPtr local_context) const; static bool supportsTotalRows(ContextPtr, ObjectStorageType) { return true; } - std::optional totalRows(ContextPtr Local_context) const override; + std::optional totalRows(ContextPtr local_context, const std::optional & state) const override; static bool supportsTotalBytes(ContextPtr, ObjectStorageType) { return true; } std::optional totalBytes(ContextPtr Local_context) const override; @@ -266,7 +275,7 @@ class IcebergMetadata : public IDataLakeMetadata std::optional sortingKey(ContextPtr) const override; private: - static Iceberg::PersistentTableComponents initializePersistentTableComponents( + static std::pair initializePersistentTableComponents( ObjectStoragePtr object_storage, StorageObjectStorageConfigurationPtr configuration, IcebergMetadataFilesCachePtr cache_ptr, @@ -332,6 +341,17 @@ class IcebergMetadata : public IDataLakeMetadata BackgroundSchedulePoolTaskHolder background_metadata_prefetch_task; ObjectIterator prepared_iterator; +<<<<<<< HEAD +======= + mutable std::mutex cached_latest_metadata_mutex; + mutable std::optional cached_latest_metadata TSA_GUARDED_BY(cached_latest_metadata_mutex); + + Iceberg::MetadataFileWithInfo fetchLatestMetadata( + const ContextPtr & context, bool force_fetch_latest_metadata, bool ignore_metadata_pointer_overrides) const; + + KeyDescription getSortingKey(ContextPtr local_context, Iceberg::TableStateSnapshot actual_table_state_snapshot) const; + +>>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) void backgroundMetadataPrefetcherThread(); void alterPartitionDropImpl( diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadataFilesCache.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadataFilesCache.h index ee0fc5c90504a..7466ee5279548 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadataFilesCache.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadataFilesCache.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -75,15 +76,20 @@ struct IcebergMetadataFilesCacheCell : private boost::noncopyable { /// The cached entities can be: /// - reference to the latest metadata version (metadata.json path) [table_path --> LatestMetadataVersionPtr] - /// - metadata.json content [file_path --> String] + /// - parsed metadata.json [file_path --> Poco::JSON::Object::Ptr] /// - manifest list consists of cache keys which will retrieve the manifest file from cache [file_path --> ManifestFileCacheKeys] /// - manifest file [file_path --> Iceberg::ManifestFileCacheableInfo] +<<<<<<< HEAD std::variant cached_element; Int64 memory_bytes; - - explicit IcebergMetadataFilesCacheCell(String && metadata_json_str) - : cached_element(std::move(metadata_json_str)) - , memory_bytes(std::get(cached_element).capacity() + SIZE_IN_MEMORY_OVERHEAD) +======= + std::variant cached_element; + size_t memory_bytes; +>>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) + + IcebergMetadataFilesCacheCell(Poco::JSON::Object::Ptr metadata_json_object, size_t metadata_json_str_size) + : cached_element(std::move(metadata_json_object)) + , memory_bytes(4 * metadata_json_str_size + SIZE_IN_MEMORY_OVERHEAD) { } explicit IcebergMetadataFilesCacheCell(LatestMetadataVersionPtr latest_metadata_version) @@ -150,19 +156,14 @@ class IcebergMetadataFilesCache : public CacheBase - String getOrSetTableMetadata(const String & data_path, LoadFunc && load_fn) + Poco::JSON::Object::Ptr getOrSetTableMetadata(const String & data_path, LoadFunc && load_fn) { - auto load_fn_wrapper = [&]() - { - auto metadata_json_str = load_fn(); - return std::make_shared(std::move(metadata_json_str)); - }; - auto result = Base::getOrSet(data_path, load_fn_wrapper); + auto result = Base::getOrSet(data_path, load_fn); if (result.second) ProfileEvents::increment(ProfileEvents::IcebergMetadataFilesCacheMisses); else ProfileEvents::increment(ProfileEvents::IcebergMetadataFilesCacheHits); - return std::get(result.first->cached_element); + return std::get(result.first->cached_element); } template diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp index 019691f05fa23..78aab4569db82 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp @@ -1510,14 +1510,14 @@ IcebergStorageSink::IcebergStorageSink( persistent_table_components.metadata_compression_method, /* ignore_explicit_metadata_file_path */ false); - metadata = getMetadataJSONObject( + metadata = deepCopy(getMetadataJSONObject( metadata_path, object_storage, persistent_table_components.metadata_cache, context, log, compression_method, - persistent_table_components.table_uuid); + persistent_table_components.table_uuid)); metadata_compression_method = compression_method; previous_metadata_file_path = metadata_path; filename_generator = FileNamesGenerator( @@ -1823,16 +1823,20 @@ bool IcebergStorageSink::initializeMetadata() previous_metadata_file_path = metadata_path; filename_generator.setVersion(last_version + 1); - metadata = getMetadataJSONObject( + metadata = deepCopy(getMetadataJSONObject( metadata_path, object_storage, persistent_table_components.metadata_cache, context, getLogger("IcebergWrites"), compression_method, +<<<<<<< HEAD persistent_table_components.table_uuid); partition_spec_id = metadata->getValue(Iceberg::f_default_spec_id); auto partitions_specs = metadata->getArray(Iceberg::f_partition_specs); +======= + persistent_table_components.table_uuid)); +>>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) auto new_schema_id = metadata->getValue(Iceberg::f_current_schema_id); if (new_schema_id != current_schema_id) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp index 9f52b73f1ca59..52dce42cbccf8 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp @@ -39,6 +39,7 @@ namespace DB namespace { +<<<<<<< HEAD Poco::JSON::Object::Ptr deepCopy(Poco::JSON::Object::Ptr obj) { std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM @@ -47,6 +48,21 @@ Poco::JSON::Object::Ptr deepCopy(Poco::JSON::Object::Ptr obj) Poco::JSON::Parser parser; auto result = parser.parse(oss.str()); return result.extract(); +======= +/// Only the incremental refreshable-MV append supplies a new cursor; every other snapshot must carry the parent's forward, else the next refresh re-appends already-materialized rows. +void carryForwardRefreshCursor( + Poco::JSON::Object::Ptr summary, Poco::JSON::Object::Ptr parent_snapshot, const std::optional & new_cursor) +{ + std::optional cursor = new_cursor; + if (!cursor.has_value() && parent_snapshot && parent_snapshot->has(Iceberg::f_summary)) + { + auto parent_summary = parent_snapshot->get(Iceberg::f_summary).extract(); + if (parent_summary->has(Iceberg::f_refresh_cursor)) + cursor = parent_summary->getValue(Iceberg::f_refresh_cursor); + } + if (cursor.has_value()) + summary->set(Iceberg::f_refresh_cursor, *cursor); +>>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) } /// Read a numeric `total-*` field from the parent snapshot's summary, returning std::nullopt when absent or null. @@ -777,7 +793,24 @@ void MetadataGenerator::generateDropColumnMetadata(const String & column_name) { const auto next_schema_id = getNextSchemaId(metadata_object); +<<<<<<< HEAD auto current_schema = deepCopy(getCurrentSchema()); +======= + Poco::JSON::Object::Ptr current_schema; + auto schemas = metadata_object->getArray(Iceberg::f_schemas); + for (UInt32 i = 0; i < schemas->size(); ++i) + { + if (schemas->getObject(i)->getValue(Iceberg::f_schema_id) == current_schema_id) + { + current_schema = schemas->getObject(i); + break; + } + } + + if (!current_schema) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Not found schema with id {}", current_schema_id); + current_schema = Iceberg::deepCopy(current_schema); +>>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) auto fields = current_schema->getArray(Iceberg::f_fields); UInt32 index_to_drop = static_cast(fields->size()); @@ -875,6 +908,7 @@ void MetadataGenerator::generateDropColumnMetadata(const String & column_name) } } +<<<<<<< HEAD current_schema->getArray(Iceberg::f_fields)->remove(index_to_drop); current_schema->set(Iceberg::f_schema_id, next_schema_id); metadata_object->set(Iceberg::f_current_schema_id, next_schema_id); @@ -889,6 +923,11 @@ void MetadataGenerator::generateAddColumnMetadata(const String & column_name, Da const auto next_schema_id = getNextSchemaId(metadata_object); auto current_schema = deepCopy(getCurrentSchema()); +======= + if (!current_schema) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Not found schema with id {}", current_schema_id); + current_schema = Iceberg::deepCopy(current_schema); +>>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) auto existing_fields = current_schema->getArray(Iceberg::f_fields); for (UInt32 i = 0; i < existing_fields->size(); ++i) @@ -955,6 +994,23 @@ bool MetadataGenerator::generateModifyColumnMetadata(const String & column_name, Iceberg::checkUnknownTypeAllowed(column_name, type, metadata_object->getValue(Iceberg::f_format_version)); auto current_schema = getCurrentSchema(); +<<<<<<< HEAD +======= + Poco::JSON::Object::Ptr current_schema; + auto schemas = metadata_object->getArray(Iceberg::f_schemas); + for (UInt32 i = 0; i < schemas->size(); ++i) + { + if (schemas->getObject(i)->getValue(Iceberg::f_schema_id) == current_schema_id) + { + current_schema = schemas->getObject(i); + break; + } + } + + if (!current_schema) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Not found schema with id {}", current_schema_id); + current_schema = Iceberg::deepCopy(current_schema); +>>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) auto last_column_id = metadata_object->getValue(Iceberg::f_last_column_id); auto format_version = metadata_object->getValue(Iceberg::f_format_version); auto schema_fields = current_schema->getArray(Iceberg::f_fields); @@ -973,12 +1029,19 @@ bool MetadataGenerator::generateModifyColumnMetadata(const String & column_name, if (current_field->getValue(Iceberg::f_name) != column_name) continue; +<<<<<<< HEAD if (current_field->getValue(Iceberg::f_required) == new_type.second && icebergTypesEqualIgnoringIds(current_field->get(Iceberg::f_type), new_type.first)) { Iceberg::IcebergSchemaProcessor schema_processor( context, context->getSettingsRef()[Setting::allow_experimental_geo_types_in_iceberg]); auto current_ch_type = schema_processor.getClickHouseFieldType(current_field, context); +======= + auto old_type = Iceberg::deepCopy(current_field); + current_field->set(Iceberg::f_type, new_type.first); + if (!current_field->getValue(Iceberg::f_required) && !type->isNullable()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Iceberg spec doesn't allow change type from nullable to non-nullable {}", type->getPrettyName()); +>>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) if (!current_ch_type->equals(*type)) throw Exception( @@ -1074,7 +1137,26 @@ bool MetadataGenerator::generateModifyColumnMetadata(const String & column_name, void MetadataGenerator::generateRenameColumnMetadata(const String & column_name, const String & new_column_name) { +<<<<<<< HEAD auto current_schema = deepCopy(getCurrentSchema()); +======= + auto current_schema_id = metadata_object->getValue(Iceberg::f_current_schema_id); + + Poco::JSON::Object::Ptr current_schema; + auto schemas = metadata_object->getArray(Iceberg::f_schemas); + for (UInt32 i = 0; i < schemas->size(); ++i) + { + if (schemas->getObject(i)->getValue(Iceberg::f_schema_id) == current_schema_id) + { + current_schema = schemas->getObject(i); + break; + } + } + + if (!current_schema) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Not found schema with id {}", current_schema_id); + current_schema = Iceberg::deepCopy(current_schema); +>>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) auto schema_fields = current_schema->getArray(Iceberg::f_fields); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp index 7d47deb9bc12d..cf064d55d0ea7 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp @@ -790,8 +790,12 @@ void mutate( filename_generator.setVersion(last_version + 1); filename_generator.setCompressionMethod(compression_method); +<<<<<<< HEAD auto metadata = getMetadataJSONObject(metadata_path, object_storage, persistent_table_components.metadata_cache, context, log, compression_method, persistent_table_components.table_uuid); auto previous_metadata_path = persistent_table_components.path_resolver.reverseResolve(metadata_path); +======= + auto metadata = deepCopy(getMetadataJSONObject(metadata_path, object_storage, persistent_table_components.metadata_cache, context, log, compression_method, persistent_table_components.table_uuid)); +>>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) /// Iceberg v3 writers must not add new position-delete files; row-level deletes require /// deletion vectors. Fail closed before any object writes until ClickHouse can write DVs. @@ -936,7 +940,7 @@ void alter( filename_generator.setVersion(last_version + 1); filename_generator.setCompressionMethod(compression_method); - auto metadata = getMetadataJSONObject(metadata_path, object_storage, persistent_table_components.metadata_cache, context, log, compression_method, persistent_table_components.table_uuid); + auto metadata = deepCopy(getMetadataJSONObject(metadata_path, object_storage, persistent_table_components.metadata_cache, context, log, compression_method, persistent_table_components.table_uuid)); const auto previous_schema_id = metadata->getValue(Iceberg::f_current_schema_id); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp index 9adc9b31ab5d4..b62476d81faa4 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp @@ -583,6 +583,93 @@ std::string normalizeUuid(const std::string & uuid) return result; } +String readMetadataJSONString( + const String & metadata_file_path, + ObjectStoragePtr object_storage, + bool use_metadata_cache, + const ContextPtr & local_context, + LoggerPtr log, + CompressionMethod compression_method) +{ + ProfileEvents::increment(ProfileEvents::IcebergMetadataJSONReads); + ProfileEventTimeIncrement watch(ProfileEvents::IcebergMetadataJSONReadMicroseconds); + + ObjectInfo object_info(metadata_file_path); + + auto read_settings = local_context->getReadSettings(); + /// Do not utilize filesystem cache if more precise cache enabled + if (use_metadata_cache) + read_settings.enable_filesystem_cache = false; + + auto source_buf = createReadBuffer(object_info.relative_path_with_metadata, object_storage, local_context, log, read_settings); + + std::unique_ptr buf; + if (compression_method != CompressionMethod::None) + /// Iceberg metadata snappy is always the Hadoop block format (`SnappyMode::Basic`); the + /// write path in `writeMessageToFile` pins the same mode. The wire format cannot be + /// inferred from the `.snappy.metadata.json` suffix, so both sides must agree statically. + buf = wrapReadBufferWithCompressionMethod(std::move(source_buf), compression_method, /*zstd_window_log_max=*/ 0, SnappyMode::Basic); + else + buf = std::move(source_buf); + + String json_str; + readJSONObjectPossiblyInvalid(json_str, *buf); + return json_str; +} + +Poco::JSON::Object::Ptr parseMetadataJSONString(const String & metadata_json_str) +{ + ProfileEvents::increment(ProfileEvents::IcebergMetadataJSONParses); + ProfileEvents::increment(ProfileEvents::IcebergMetadataJSONParsedBytes, metadata_json_str.size()); + ProfileEventTimeIncrement watch(ProfileEvents::IcebergMetadataJSONParseMicroseconds); + + Poco::JSON::Parser parser; /// For some reason base/base/JSON.h can not parse this json file + Poco::Dynamic::Var json = parser.parse(metadata_json_str); + return json.extract(); +} + +static Poco::Dynamic::Var deepCopyJSONValue(const Poco::Dynamic::Var & value); + +static Poco::JSON::Array::Ptr deepCopy(const Poco::JSON::Array::Ptr & array) +{ + Poco::JSON::Array::Ptr result = new Poco::JSON::Array(array->getEscapeUnicode() ? Poco::JSON_ESCAPE_UNICODE : 0); + for (const auto & element : *array) + result->add(deepCopyJSONValue(element)); + return result; +} + +static bool isJSONContainer(const Poco::Dynamic::Var & value) +{ + const auto & type = value.type(); + return type == typeid(Poco::JSON::Object::Ptr) || type == typeid(Poco::JSON::Array::Ptr) + || type == typeid(Poco::JSON::Object) || type == typeid(Poco::JSON::Array); +} + +static Poco::Dynamic::Var deepCopyJSONValue(const Poco::Dynamic::Var & value) +{ + const auto & type = value.type(); + if (type == typeid(Poco::JSON::Object::Ptr)) + return deepCopy(value.extract()); + if (type == typeid(Poco::JSON::Array::Ptr)) + return deepCopy(value.extract()); + if (type == typeid(Poco::JSON::Object)) + return deepCopy(Poco::JSON::Object::Ptr(new Poco::JSON::Object(value.extract()))); + if (type == typeid(Poco::JSON::Array)) + return deepCopy(Poco::JSON::Array::Ptr(new Poco::JSON::Array(value.extract()))); + return value; +} + +Poco::JSON::Object::Ptr deepCopy(const Poco::JSON::Object::Ptr & object) +{ + Poco::JSON::Object::Ptr result = new Poco::JSON::Object(*object); + for (const auto & [key, value] : *object) + { + if (isJSONContainer(value)) + result->set(key, deepCopyJSONValue(value)); + } + return result; +} + Poco::JSON::Object::Ptr getMetadataJSONObject( const String & metadata_file_path, ObjectStoragePtr object_storage, @@ -592,8 +679,9 @@ Poco::JSON::Object::Ptr getMetadataJSONObject( CompressionMethod compression_method, const std::optional & table_uuid) { - auto create_fn = [&]() + auto read_fn = [&]() { +<<<<<<< HEAD ObjectInfo object_info(metadata_file_path); auto read_settings = local_context->getReadSettings(); @@ -630,6 +718,21 @@ Poco::JSON::Object::Ptr getMetadataJSONObject( Poco::JSON::Parser parser; /// For some reason base/base/JSON.h can not parse this json file Poco::Dynamic::Var json = parser.parse(metadata_json_str); return json.extract(); +======= + return readMetadataJSONString(metadata_file_path, object_storage, metadata_cache != nullptr, local_context, log, compression_method); + }; + + if (!metadata_cache || !table_uuid.has_value()) + return parseMetadataJSONString(read_fn()); + + return metadata_cache->getOrSetTableMetadata( + IcebergMetadataFilesCache::getKey(*table_uuid, metadata_file_path), + [&]() + { + auto metadata_json_str = read_fn(); + return std::make_shared(parseMetadataJSONString(metadata_json_str), metadata_json_str.size()); + }); +>>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) } /// Iceberg stores a decimal as its unscaled value in two's-complement big-endian form, in the diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h index b58a9f411c2a1..c06cfe0fb8340 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h @@ -113,6 +113,18 @@ Poco::JSON::Object::Ptr getMetadataJSONObject( CompressionMethod compression_method, const std::optional & table_uuid); +String readMetadataJSONString( + const String & metadata_file_path, + ObjectStoragePtr object_storage, + bool use_metadata_cache, + const ContextPtr & local_context, + LoggerPtr log, + CompressionMethod compression_method); + +Poco::JSON::Object::Ptr parseMetadataJSONString(const String & metadata_json_str); + +Poco::JSON::Object::Ptr deepCopy(const Poco::JSON::Object::Ptr & object); + /// Maps a ClickHouse type to an Iceberg JSON type. `timestamp_ns` / `timestamptz_ns` /// require `format_version` >= 3; lower versions throw instead of writing v3 types diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.cpp b/src/Storages/ObjectStorage/StorageObjectStorage.cpp index 1d72308cd3b4e..b9e9d1e3ed8bf 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorage.cpp @@ -557,6 +557,11 @@ void StorageObjectStorage::updateExternalDynamicMetadataIfExists(ContextPtr quer std::optional StorageObjectStorage::totalRows(ContextPtr query_context) const +{ + return totalRowsInSnapshot(nullptr, query_context); +} + +std::optional StorageObjectStorage::totalRowsInSnapshot(const StorageSnapshotPtr & storage_snapshot, ContextPtr query_context) const { if (!configuration->supportsTotalRows(query_context, object_storage->getType())) return std::nullopt; @@ -567,9 +572,16 @@ std::optional StorageObjectStorage::totalRows(ContextPtr query_context) if (distributed_processing) return std::nullopt; - is_table_function ? configuration->lazyInitializeIfNeeded(object_storage, query_context) : configuration->update(object_storage, query_context); + std::optional state; + if (storage_snapshot && storage_snapshot->metadata) + state = storage_snapshot->metadata->datalake_table_state; + + if (is_table_function || state) + configuration->lazyInitializeIfNeeded(object_storage, query_context); + else + configuration->update(object_storage, query_context); - return configuration->totalRows(query_context); + return configuration->totalRows(query_context, state); } std::optional StorageObjectStorage::totalBytes(ContextPtr query_context) const diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.h b/src/Storages/ObjectStorage/StorageObjectStorage.h index 15c41e5f2e235..1f31e30e01a28 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.h +++ b/src/Storages/ObjectStorage/StorageObjectStorage.h @@ -211,6 +211,7 @@ class StorageObjectStorage : public IStorage, public IBackgroundOperation std::shared_ptr getCatalog() const { return catalog; } std::optional totalRows(ContextPtr query_context) const override; + std::optional totalRowsInSnapshot(const StorageSnapshotPtr & storage_snapshot, ContextPtr query_context) const override; std::optional totalBytes(ContextPtr query_context) const override; bool optimize( diff --git a/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp b/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp index 8e5bcb35af231..3cbcc789c7b59 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp @@ -313,12 +313,25 @@ void StorageObjectStorageCluster::prepareForDrop(ContextPtr query_context) std::optional StorageObjectStorageCluster::totalRows(ContextPtr query_context) const { +<<<<<<< HEAD if (pure_storage) return pure_storage->totalRows(query_context); +======= + return totalRowsInSnapshot(nullptr, query_context); +} + +std::optional StorageObjectStorageCluster::totalRowsInSnapshot(const StorageSnapshotPtr & storage_snapshot, ContextPtr query_context) const +{ +>>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) configuration->lazyInitializeIfNeeded( object_storage, query_context); - return configuration->totalRows(query_context); + + std::optional state; + if (storage_snapshot && storage_snapshot->metadata) + state = storage_snapshot->metadata->datalake_table_state; + + return configuration->totalRows(query_context, state); } std::optional StorageObjectStorageCluster::totalBytes(ContextPtr query_context) const diff --git a/src/Storages/ObjectStorage/StorageObjectStorageCluster.h b/src/Storages/ObjectStorage/StorageObjectStorageCluster.h index ccec7515171a9..8e68684aa99ed 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageCluster.h +++ b/src/Storages/ObjectStorage/StorageObjectStorageCluster.h @@ -64,6 +64,7 @@ class StorageObjectStorageCluster : public IStorageCluster String getPathSample(ContextPtr context); std::optional totalRows(ContextPtr query_context) const override; + std::optional totalRowsInSnapshot(const StorageSnapshotPtr & storage_snapshot, ContextPtr query_context) const override; std::optional totalBytes(ContextPtr query_context) const override; void setClusterNameFromFunctionArgument(bool cluster_name_from_function_argument_) diff --git a/src/Storages/ObjectStorage/StorageObjectStorageConfiguration.h b/src/Storages/ObjectStorage/StorageObjectStorageConfiguration.h index b4fc661949249..0703b35d739a9 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageConfiguration.h +++ b/src/Storages/ObjectStorage/StorageObjectStorageConfiguration.h @@ -155,7 +155,7 @@ class StorageObjectStorageConfiguration virtual bool isIcebergConfiguration() const { return false; } virtual bool supportsTotalRows(ContextPtr, ObjectStorageType) const { return false; } - virtual std::optional totalRows(ContextPtr) { return {}; } + virtual std::optional totalRows(ContextPtr, const std::optional &) { return {}; } virtual bool supportsTotalBytes(ContextPtr, ObjectStorageType) const { return false; } virtual std::optional totalBytes(ContextPtr) { return {}; } /// NOTE: In this function we are going to check is data which we are going to read sorted by sorting key specified in StorageMetadataPtr. diff --git a/tests/queries/0_stateless/05407_iceberg_metadata_s3_requests.reference b/tests/queries/0_stateless/05407_iceberg_metadata_s3_requests.reference new file mode 100644 index 0000000000000..814de72cada12 --- /dev/null +++ b/tests/queries/0_stateless/05407_iceberg_metadata_s3_requests.reference @@ -0,0 +1,4 @@ +table_function_select 1 1 +table_function_count 1 1 +table_select 1 0 +table_count 1 0 diff --git a/tests/queries/0_stateless/05407_iceberg_metadata_s3_requests.sh b/tests/queries/0_stateless/05407_iceberg_metadata_s3_requests.sh new file mode 100755 index 0000000000000..5a0e3cce028f2 --- /dev/null +++ b/tests/queries/0_stateless/05407_iceberg_metadata_s3_requests.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +TABLE="t_${CLICKHOUSE_DATABASE}_${RANDOM}" +TABLE_PATH="05407_iceberg_metadata_s3_requests/${CLICKHOUSE_TEST_UNIQUE_NAME}" + +${CLICKHOUSE_CLIENT} --query "CREATE TABLE ${TABLE} (c0 Int32) ENGINE = IcebergS3(s3_conn, filename = '${TABLE_PATH}')" + +run_query() +{ + ${CLICKHOUSE_CLIENT} --log_comment="${CLICKHOUSE_TEST_UNIQUE_NAME}_$1" \ + --query "$2 SETTINGS use_iceberg_metadata_files_cache = 1, iceberg_metadata_staleness_ms = 0" > /dev/null +} + +run_query table_function_select "SELECT * FROM icebergS3(s3_conn, filename = '${TABLE_PATH}')" +run_query table_function_count "SELECT count() FROM icebergS3(s3_conn, filename = '${TABLE_PATH}')" +run_query table_select "SELECT * FROM ${TABLE}" +run_query table_count "SELECT count() FROM ${TABLE}" + +${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS query_log" +${CLICKHOUSE_CLIENT} --query " + SELECT replaceOne(log_comment, '${CLICKHOUSE_TEST_UNIQUE_NAME}_', ''), ProfileEvents['S3ListObjects'], ProfileEvents['S3GetObject'] + FROM system.query_log + WHERE current_database = currentDatabase() AND type = 'QueryFinish' + AND log_comment LIKE '${CLICKHOUSE_TEST_UNIQUE_NAME}\_%' + ORDER BY event_time_microseconds +" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE ${TABLE}" diff --git a/tests/queries/0_stateless/05408_iceberg_rejected_alter_keeps_cached_metadata.reference b/tests/queries/0_stateless/05408_iceberg_rejected_alter_keeps_cached_metadata.reference new file mode 100644 index 0000000000000..c34437db65303 --- /dev/null +++ b/tests/queries/0_stateless/05408_iceberg_rejected_alter_keeps_cached_metadata.reference @@ -0,0 +1,4 @@ +6 +BAD_ARGUMENTS +6 +c0 Nullable(Int64) diff --git a/tests/queries/0_stateless/05408_iceberg_rejected_alter_keeps_cached_metadata.sh b/tests/queries/0_stateless/05408_iceberg_rejected_alter_keeps_cached_metadata.sh new file mode 100755 index 0000000000000..4ca5026de551d --- /dev/null +++ b/tests/queries/0_stateless/05408_iceberg_rejected_alter_keeps_cached_metadata.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +TABLE="t_${CLICKHOUSE_DATABASE}_${RANDOM}" +TABLE_PATH="${USER_FILES_PATH}/${TABLE}/" + +trap 'rm -rf "${TABLE_PATH}" 2>/dev/null' EXIT + +CLIENT="${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=1 --allow_insert_into_iceberg=1" + +${CLIENT} --query "CREATE TABLE ${TABLE} (c0 Nullable(Int64)) ENGINE = IcebergLocal('${TABLE_PATH}')" +${CLIENT} --query "INSERT INTO ${TABLE} VALUES (1), (2), (3)" +${CLIENT} --query "SELECT sum(c0) FROM ${TABLE}" + +${CLIENT} --query "ALTER TABLE ${TABLE} MODIFY COLUMN c0 Int64" 2>&1 | grep -o -m1 "BAD_ARGUMENTS" + +${CLIENT} --query "SELECT sum(c0) FROM ${TABLE}" +${CLIENT} --query "SELECT name, type FROM system.columns WHERE database = currentDatabase() AND table = '${TABLE}'" + +${CLIENT} --query "DROP TABLE ${TABLE}" diff --git a/tests/queries/0_stateless/05409_iceberg_select_sees_rows_appended_after_create.reference b/tests/queries/0_stateless/05409_iceberg_select_sees_rows_appended_after_create.reference new file mode 100644 index 0000000000000..35ab43b0eb62a --- /dev/null +++ b/tests/queries/0_stateless/05409_iceberg_select_sees_rows_appended_after_create.reference @@ -0,0 +1 @@ +3 6 diff --git a/tests/queries/0_stateless/05409_iceberg_select_sees_rows_appended_after_create.sh b/tests/queries/0_stateless/05409_iceberg_select_sees_rows_appended_after_create.sh new file mode 100755 index 0000000000000..95e774539c05d --- /dev/null +++ b/tests/queries/0_stateless/05409_iceberg_select_sees_rows_appended_after_create.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +TABLE="t_${CLICKHOUSE_DATABASE}_${RANDOM}" +TABLE_PATH="${USER_FILES_PATH}/${TABLE}/" + +trap 'rm -rf "${TABLE_PATH}" 2>/dev/null' EXIT + +${CLICKHOUSE_CLIENT} --query "CREATE TABLE ${TABLE} (c0 Int64) ENGINE = IcebergLocal('${TABLE_PATH}')" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "INSERT INTO FUNCTION icebergLocal('${TABLE_PATH}') VALUES (1), (2), (3)" +${CLICKHOUSE_CLIENT} --iceberg_metadata_staleness_ms=0 --query "SELECT count(), sum(c0) FROM ${TABLE}" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE ${TABLE}" diff --git a/tests/queries/0_stateless/05441_iceberg_timestamp_selection_lower_version_replaces_cached.reference b/tests/queries/0_stateless/05441_iceberg_timestamp_selection_lower_version_replaces_cached.reference new file mode 100644 index 0000000000000..db47195262f31 --- /dev/null +++ b/tests/queries/0_stateless/05441_iceberg_timestamp_selection_lower_version_replaces_cached.reference @@ -0,0 +1,2 @@ +21 +6 diff --git a/tests/queries/0_stateless/05441_iceberg_timestamp_selection_lower_version_replaces_cached.sh b/tests/queries/0_stateless/05441_iceberg_timestamp_selection_lower_version_replaces_cached.sh new file mode 100755 index 0000000000000..26f96d83269b6 --- /dev/null +++ b/tests/queries/0_stateless/05441_iceberg_timestamp_selection_lower_version_replaces_cached.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +TABLE="t_${CLICKHOUSE_DATABASE}_${RANDOM}" +TABLE_PATH="${USER_FILES_PATH}/${TABLE}/" + +function cleanup() +{ + rm -rf "${TABLE_PATH}" 2>/dev/null +} +trap cleanup EXIT + +CLIENT="${CLICKHOUSE_CLIENT} --iceberg_metadata_staleness_ms=0 --allow_insert_into_iceberg=1" + +${CLIENT} --query "CREATE TABLE ${TABLE} (c0 Int64) ENGINE = IcebergLocal('${TABLE_PATH}') SETTINGS iceberg_recent_metadata_file_by_last_updated_ms_field = 1" +${CLIENT} --query "INSERT INTO ${TABLE} VALUES (1), (2), (3)" +${CLIENT} --query "INSERT INTO ${TABLE} VALUES (4), (5), (6)" +${CLIENT} --query "SELECT sum(c0) FROM ${TABLE}" + +sed -E 's/"last-updated-ms" : [0-9]+/"last-updated-ms" : 9999999999999/' \ + "${TABLE_PATH}/metadata/v2.metadata.json" > "${TABLE_PATH}/metadata/v0.metadata.json" + +${CLIENT} --query "SELECT sum(c0) FROM ${TABLE}" + +${CLIENT} --query "DROP TABLE ${TABLE}" From 34061b6a2586aacda05c4c229c535148523293f8 Mon Sep 17 00:00:00 2001 From: blau-ai <302218339+blau-ai@users.noreply.github.com> Date: Mon, 16 Nov 2026 09:17:02 +0000 Subject: [PATCH 2/2] Resolve conflicts in backport of #126267 Resolved 10 conflicts by keeping the 26.8 code and applying the PR's metadata-lookup and parsed-JSON caching; dropped the `ignore_metadata_pointer_overrides`, `explicit_metadata_file_path` and catalog-lookup code and the `createInitial` declaration, which 26.8 lacks. --- src/Planner/PlannerJoinTree.cpp | 19 ---- .../DataLakes/DataLakeConfiguration.h | 2 +- .../DataLakes/DeltaLakeMetadataDeltaKernel.h | 17 --- .../DataLakes/Iceberg/BinPackRewrite.cpp | 8 +- .../DataLakes/Iceberg/IcebergMetadata.cpp | 82 +++----------- .../DataLakes/Iceberg/IcebergMetadata.h | 14 +-- .../Iceberg/IcebergMetadataFilesCache.h | 9 +- .../DataLakes/Iceberg/IcebergWrites.cpp | 6 +- .../DataLakes/Iceberg/MetadataGenerator.cpp | 100 +----------------- .../DataLakes/Iceberg/Mutations.cpp | 6 +- .../ObjectStorage/DataLakes/Iceberg/Utils.cpp | 47 +------- .../StorageObjectStorageCluster.cpp | 7 +- 12 files changed, 31 insertions(+), 286 deletions(-) diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index e247d321e9b55..dd8d9d750f5b3 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -780,28 +780,9 @@ bool applyTrivialCountIfPossible( /// Set a special flag in query info so the storage will see it and optimize count in read() method. select_query_info.optimize_trivial_count = true; -<<<<<<< HEAD /// Get number of rows - std::optional num_rows = storage->totalRows(query_context); -======= - /// Do not apply trivial count optimization if we're the follower. Because if every follower blindly returns totalRows(), - /// the result will be R times the actual count of rows, where R is the number of replicas. - /// It's important to make this check after we've set the `select_query_info.optimize_trivial_count` flag, so that storage could apply - /// some optimization during the read phase. - /// TODO: We could still avoid reading data for MergeTree on the followers: we could read the parts metadata for the set of parts - /// we were handed out by the initiator. This would require consuming the `select_query_info.optimize_trivial_count` during the read - /// phase and acting accordingly. - if (query_context->canUseParallelReplicasOnFollower()) - return false; - - /// Get number of rows. - /// MergeTree maintains total count for all parts in Active state and it simply returns that number for trivial - /// `select count() from table` query. But if we have current transaction, then we should return number of rows in current snapshot - /// (that may include parts in Outdated state), so the totalRows for transactional case should return the number of rows visible for - /// the current transaction. const auto & storage_snapshot = table_node ? table_node->getStorageSnapshot() : table_function_node->getStorageSnapshot(); std::optional num_rows = storage->totalRowsInSnapshot(storage_snapshot, query_context); ->>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) if (!num_rows) return false; diff --git a/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h b/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h index 2a23cea3dac75..9a0bb613f5204 100644 --- a/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h +++ b/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h @@ -661,7 +661,7 @@ class StorageIcebergConfiguration : public StorageObjectStorageConfiguration, pu bool isDataLakeConfiguration() const override { return getImpl().isDataLakeConfiguration(); } bool supportsTotalRows(ContextPtr context, ObjectStorageType storage_type) const override { return getImpl().supportsTotalRows(context, storage_type); } - std::optional totalRows(ContextPtr context) override { return getImpl().totalRows(context); } + std::optional totalRows(ContextPtr context, const std::optional & state) override { return getImpl().totalRows(context, state); } bool supportsTotalBytes(ContextPtr context, ObjectStorageType storage_type) const override { return getImpl().supportsTotalBytes(context, storage_type); } std::optional totalBytes(ContextPtr context) override { return getImpl().totalBytes(context); } bool isDataSortedBySortingKey(StorageMetadataPtr storage_metadata, ContextPtr context) const override diff --git a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadataDeltaKernel.h b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadataDeltaKernel.h index 214b5ceb38a1a..6f59403ffcfab 100644 --- a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadataDeltaKernel.h +++ b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadataDeltaKernel.h @@ -69,24 +69,7 @@ class DeltaLakeMetadataDeltaKernel final : public IDataLakeMetadata return std::make_unique(object_storage_, configuration); } -<<<<<<< HEAD - std::optional totalRows(ContextPtr) const override; -======= - /// CREATE TABLE entry point when the kernel is enabled: validate the declared columns, write commit 0 - /// (fresh) or attach to an existing `_delta_log`, and register the table in the catalog when present. - static void createInitial( - const ObjectStoragePtr & object_storage, - const StorageObjectStorageConfigurationWeakPtr & configuration, - const ContextPtr & local_context, - const std::optional & columns, - ASTPtr partition_by, - ASTPtr order_by, - bool if_not_exists, - std::shared_ptr catalog, - const StorageID & table_id_); - std::optional totalRows(ContextPtr, const std::optional &) const override; ->>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) std::optional totalBytes(ContextPtr) const override; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/BinPackRewrite.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/BinPackRewrite.cpp index aec7e2be05ffd..b3801870a9a84 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/BinPackRewrite.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/BinPackRewrite.cpp @@ -649,14 +649,14 @@ LatestMetadata readLatestMetadata( /* force_fetch_latest_metadata */ true, /* ignore_explicit_metadata_file_path */ false); - auto metadata_object = getMetadataJSONObject( + auto metadata_object = deepCopy(getMetadataJSONObject( catalog_metadata.path, object_storage, persistent_table_components.metadata_cache, context, log, catalog_metadata.compression_method, - persistent_table_components.table_uuid); + persistent_table_components.table_uuid)); /// The new metadata file must not reuse the name of a newer file left in the directory. const auto max_directory_version = getMaxMetadataFileVersionInDirectory(object_storage, persistent_table_components.table_path); @@ -677,14 +677,14 @@ LatestMetadata readLatestMetadata( /* force_fetch_latest_metadata */ true, /* ignore_explicit_metadata_file_path */ true); - auto metadata_object = getMetadataJSONObject( + auto metadata_object = deepCopy(getMetadataJSONObject( metadata_file_path, object_storage, persistent_table_components.metadata_cache, context, log, persistent_table_components.metadata_compression_method, - persistent_table_components.table_uuid); + persistent_table_components.table_uuid)); return {metadata_version, metadata_object, {}}; } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index 269287fe046a7..b54e276afe074 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -280,14 +280,9 @@ std::pair Ice [&]() { return std::make_shared(metadata_object, metadata_json_str.size()); }); auto table_path = configuration->getPathForRead().path; auto root_derivation = IcebergPathResolver::deriveTableRoot(table_location, table_path, metadata_file_path); -<<<<<<< HEAD - return PersistentTableComponents{ + PersistentTableComponents persistent_components{ .schema_processor = std::make_shared( context_, context_->getSettingsRef()[Setting::allow_experimental_geo_types_in_iceberg]), -======= - PersistentTableComponents persistent_components{ - .schema_processor = std::make_shared(context_->getSettingsRef()[Setting::allow_geo_types_in_iceberg]), ->>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) .metadata_cache = cache_ptr, .format_version = format_version, .table_location = table_location, @@ -302,54 +297,30 @@ std::pair Ice return {std::move(persistent_components), latest_metadata}; } -<<<<<<< HEAD -======= -void IcebergMetadata::setExplicitMetadataFilePath(const String & path) -{ - explicit_metadata_file_path.set(std::make_unique(path)); -} - void IcebergMetadata::update(const ContextPtr &) { std::lock_guard lock(cached_latest_metadata_mutex); cached_latest_metadata.reset(); } -Iceberg::MetadataFileWithInfo IcebergMetadata::fetchLatestMetadata( - const ContextPtr & context, bool force_fetch_latest_metadata, bool ignore_metadata_pointer_overrides) const +Iceberg::MetadataFileWithInfo IcebergMetadata::fetchLatestMetadata(const ContextPtr & context, bool force_fetch_latest_metadata) const { return getLatestOrExplicitMetadataFileAndVersion( object_storage, persistent_components.table_path, - getMetadataLookupSettings(), + data_lake_settings, persistent_components.metadata_cache, context, log.get(), persistent_components.table_uuid, persistent_components.metadata_compression_method, - force_fetch_latest_metadata, - ignore_metadata_pointer_overrides); -} - -DataLakeStorageSettings IcebergMetadata::getMetadataLookupSettings() const -{ - DataLakeStorageSettings result = data_lake_settings; - if (auto path = explicit_metadata_file_path.get()) - result[DataLakeStorageSetting::iceberg_metadata_file_path] = *path; - return result; + force_fetch_latest_metadata); } ->>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) std::pair IcebergMetadata::getRelevantState( const ContextPtr & context, bool force_fetch_latest_metadata) const { -<<<<<<< HEAD - const auto [metadata_version, metadata_file_path, compression_method] = getLatestOrExplicitMetadataFileAndVersion( - object_storage, -======= - ProfileEventTimeIncrement watch(ProfileEvents::IcebergGetTableStateMicroseconds); - if (!force_fetch_latest_metadata && !ignore_metadata_pointer_overrides && !explicit_metadata_file_path.get() - && !data_lake_settings[DataLakeStorageSetting::iceberg_metadata_file_path].changed) + if (!force_fetch_latest_metadata && !data_lake_settings[DataLakeStorageSetting::iceberg_metadata_file_path].changed) { std::optional latest_metadata; { @@ -360,30 +331,10 @@ std::pair IcebergMetadata::getReleva return getState(context, latest_metadata->path, latest_metadata->version); } - const auto latest_metadata = fetchLatestMetadata(context, force_fetch_latest_metadata, ignore_metadata_pointer_overrides); + const auto latest_metadata = fetchLatestMetadata(context, force_fetch_latest_metadata); return getState(context, latest_metadata.path, latest_metadata.version); } -std::pair IcebergMetadata::getRelevantState( - const ContextPtr & context, const std::shared_ptr & catalog, const String & table_identifier) const -{ - ProfileEventTimeIncrement watch(ProfileEvents::IcebergGetTableStateMicroseconds); - const auto [metadata_version, metadata_file_path, _] = getLatestMetadataFileAndVersionWithCatalog( - object_storage, - catalog, - table_identifier, ->>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) - persistent_components.table_path, - data_lake_settings, - persistent_components.metadata_cache, - context, - log.get(), - persistent_components.table_uuid, - persistent_components.metadata_compression_method, - force_fetch_latest_metadata); - return getState(context, metadata_file_path, metadata_version); -} - IcebergMetadata::IcebergMetadata( ObjectStoragePtr object_storage_, StorageObjectStorageConfigurationPtr configuration_, @@ -395,12 +346,8 @@ IcebergMetadata::IcebergMetadata( , external_storages(std::make_shared()) , persistent_components(std::move(persistent_components_)) , data_lake_settings(configuration_->getDataLakeSettings()) -<<<<<<< HEAD , write_format(configuration_->getFormat()) -======= - , write_format(configuration_->format) , cached_latest_metadata(std::move(cached_latest_metadata_)) ->>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) { /// TODO: for now it's okay to start/stop the task via constructor/destructor. Once refactored, we'd need to plumb startup/shutdown and schedule the task from there if (persistent_components.metadata_cache && data_lake_settings[DataLakeStorageSetting::iceberg_metadata_async_prefetch_period_ms] != 0) @@ -945,14 +892,14 @@ void IcebergMetadata::truncate(ContextPtr context, std::shared_ptrgetTypeName(), configuration_ptr->getNamespace()), /// Consistent with the resolver above, which is rooted at `table_path` itself. -<<<<<<< HEAD .table_root_was_derived = false, .common_namespace = configuration_ptr->getNamespace()}; - auto metadata = std::make_unique(object_storage, configuration.lock(), std::move(deserialized_persistent_components), local_context); -======= - .table_root_was_derived = false}; auto metadata = std::make_unique(object_storage, configuration.lock(), std::move(deserialized_persistent_components), local_context, latest_metadata); ->>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) return metadata; } @@ -2519,14 +2461,14 @@ std::optional IcebergMetadata::commitImport filename_generator.setVersion(last_version + 1); filename_generator.setCompressionMethod(compression_method); - metadata = getMetadataJSONObject( + metadata = deepCopy(getMetadataJSONObject( metadata_path, object_storage, persistent_components.metadata_cache, context, getLogger("IcebergMetadata"), compression_method, - persistent_components.table_uuid); + persistent_components.table_uuid)); metadata_file_path = metadata_path; /// For the export path the schema and partition spec are fixed at the start of the @@ -2758,14 +2700,14 @@ IStorage::ExportPartitionCommitInfo IcebergMetadata::commitExportPartitionTransa true); /// Latest metadata is ALWAYS necessary to commit - but we abort in case schema or partition spec changed - Poco::JSON::Object::Ptr metadata = getMetadataJSONObject( + Poco::JSON::Object::Ptr metadata = deepCopy(getMetadataJSONObject( updated_metadata_file_info.path, object_storage, persistent_components.metadata_cache, context, getLogger("IcebergMetadata"), updated_metadata_file_info.compression_method, - persistent_components.table_uuid); + persistent_components.table_uuid)); if (isExportPartitionTransactionAlreadyCommitted(metadata, transaction_id)) { diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h index 40b9bf4cd01bd..f365d5af9f077 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -114,14 +115,9 @@ class IcebergMetadata : public IDataLakeMetadata LoggerPtr metadata_logger); bool supportsUpdate() const override { return true; } -<<<<<<< HEAD -======= void update(const ContextPtr & context) override; - void setExplicitMetadataFilePath(const String & path) override; - ->>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) bool supportsWrites() const override { return true; } bool supportsParallelInsert() const override { return true; } bool supportsTruncate() const override { return true; } @@ -341,17 +337,11 @@ class IcebergMetadata : public IDataLakeMetadata BackgroundSchedulePoolTaskHolder background_metadata_prefetch_task; ObjectIterator prepared_iterator; -<<<<<<< HEAD -======= mutable std::mutex cached_latest_metadata_mutex; mutable std::optional cached_latest_metadata TSA_GUARDED_BY(cached_latest_metadata_mutex); - Iceberg::MetadataFileWithInfo fetchLatestMetadata( - const ContextPtr & context, bool force_fetch_latest_metadata, bool ignore_metadata_pointer_overrides) const; - - KeyDescription getSortingKey(ContextPtr local_context, Iceberg::TableStateSnapshot actual_table_state_snapshot) const; + Iceberg::MetadataFileWithInfo fetchLatestMetadata(const ContextPtr & context, bool force_fetch_latest_metadata) const; ->>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) void backgroundMetadataPrefetcherThread(); void alterPartitionDropImpl( diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadataFilesCache.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadataFilesCache.h index 7466ee5279548..89be67ede2b65 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadataFilesCache.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadataFilesCache.h @@ -79,17 +79,12 @@ struct IcebergMetadataFilesCacheCell : private boost::noncopyable /// - parsed metadata.json [file_path --> Poco::JSON::Object::Ptr] /// - manifest list consists of cache keys which will retrieve the manifest file from cache [file_path --> ManifestFileCacheKeys] /// - manifest file [file_path --> Iceberg::ManifestFileCacheableInfo] -<<<<<<< HEAD - std::variant cached_element; - Int64 memory_bytes; -======= std::variant cached_element; - size_t memory_bytes; ->>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) + Int64 memory_bytes; IcebergMetadataFilesCacheCell(Poco::JSON::Object::Ptr metadata_json_object, size_t metadata_json_str_size) : cached_element(std::move(metadata_json_object)) - , memory_bytes(4 * metadata_json_str_size + SIZE_IN_MEMORY_OVERHEAD) + , memory_bytes(static_cast(4 * metadata_json_str_size + SIZE_IN_MEMORY_OVERHEAD)) { } explicit IcebergMetadataFilesCacheCell(LatestMetadataVersionPtr latest_metadata_version) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp index 78aab4569db82..af0a0ce01c9a2 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp @@ -1830,13 +1830,9 @@ bool IcebergStorageSink::initializeMetadata() context, getLogger("IcebergWrites"), compression_method, -<<<<<<< HEAD - persistent_table_components.table_uuid); + persistent_table_components.table_uuid)); partition_spec_id = metadata->getValue(Iceberg::f_default_spec_id); auto partitions_specs = metadata->getArray(Iceberg::f_partition_specs); -======= - persistent_table_components.table_uuid)); ->>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) auto new_schema_id = metadata->getValue(Iceberg::f_current_schema_id); if (new_schema_id != current_schema_id) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp index 52dce42cbccf8..65a1f669942d4 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp @@ -39,32 +39,6 @@ namespace DB namespace { -<<<<<<< HEAD -Poco::JSON::Object::Ptr deepCopy(Poco::JSON::Object::Ptr obj) -{ - std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - obj->stringify(oss); - - Poco::JSON::Parser parser; - auto result = parser.parse(oss.str()); - return result.extract(); -======= -/// Only the incremental refreshable-MV append supplies a new cursor; every other snapshot must carry the parent's forward, else the next refresh re-appends already-materialized rows. -void carryForwardRefreshCursor( - Poco::JSON::Object::Ptr summary, Poco::JSON::Object::Ptr parent_snapshot, const std::optional & new_cursor) -{ - std::optional cursor = new_cursor; - if (!cursor.has_value() && parent_snapshot && parent_snapshot->has(Iceberg::f_summary)) - { - auto parent_summary = parent_snapshot->get(Iceberg::f_summary).extract(); - if (parent_summary->has(Iceberg::f_refresh_cursor)) - cursor = parent_summary->getValue(Iceberg::f_refresh_cursor); - } - if (cursor.has_value()) - summary->set(Iceberg::f_refresh_cursor, *cursor); ->>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) -} - /// Read a numeric `total-*` field from the parent snapshot's summary, returning std::nullopt when absent or null. std::optional readParentTotal(Poco::JSON::Object::Ptr parent_snapshot, const char * field_name) { @@ -793,24 +767,7 @@ void MetadataGenerator::generateDropColumnMetadata(const String & column_name) { const auto next_schema_id = getNextSchemaId(metadata_object); -<<<<<<< HEAD - auto current_schema = deepCopy(getCurrentSchema()); -======= - Poco::JSON::Object::Ptr current_schema; - auto schemas = metadata_object->getArray(Iceberg::f_schemas); - for (UInt32 i = 0; i < schemas->size(); ++i) - { - if (schemas->getObject(i)->getValue(Iceberg::f_schema_id) == current_schema_id) - { - current_schema = schemas->getObject(i); - break; - } - } - - if (!current_schema) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Not found schema with id {}", current_schema_id); - current_schema = Iceberg::deepCopy(current_schema); ->>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) + auto current_schema = Iceberg::deepCopy(getCurrentSchema()); auto fields = current_schema->getArray(Iceberg::f_fields); UInt32 index_to_drop = static_cast(fields->size()); @@ -908,7 +865,6 @@ void MetadataGenerator::generateDropColumnMetadata(const String & column_name) } } -<<<<<<< HEAD current_schema->getArray(Iceberg::f_fields)->remove(index_to_drop); current_schema->set(Iceberg::f_schema_id, next_schema_id); metadata_object->set(Iceberg::f_current_schema_id, next_schema_id); @@ -922,12 +878,7 @@ void MetadataGenerator::generateAddColumnMetadata(const String & column_name, Da Iceberg::checkUnknownTypeAllowed(column_name, type, metadata_object->getValue(Iceberg::f_format_version)); const auto next_schema_id = getNextSchemaId(metadata_object); - auto current_schema = deepCopy(getCurrentSchema()); -======= - if (!current_schema) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Not found schema with id {}", current_schema_id); - current_schema = Iceberg::deepCopy(current_schema); ->>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) + auto current_schema = Iceberg::deepCopy(getCurrentSchema()); auto existing_fields = current_schema->getArray(Iceberg::f_fields); for (UInt32 i = 0; i < existing_fields->size(); ++i) @@ -994,23 +945,6 @@ bool MetadataGenerator::generateModifyColumnMetadata(const String & column_name, Iceberg::checkUnknownTypeAllowed(column_name, type, metadata_object->getValue(Iceberg::f_format_version)); auto current_schema = getCurrentSchema(); -<<<<<<< HEAD -======= - Poco::JSON::Object::Ptr current_schema; - auto schemas = metadata_object->getArray(Iceberg::f_schemas); - for (UInt32 i = 0; i < schemas->size(); ++i) - { - if (schemas->getObject(i)->getValue(Iceberg::f_schema_id) == current_schema_id) - { - current_schema = schemas->getObject(i); - break; - } - } - - if (!current_schema) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Not found schema with id {}", current_schema_id); - current_schema = Iceberg::deepCopy(current_schema); ->>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) auto last_column_id = metadata_object->getValue(Iceberg::f_last_column_id); auto format_version = metadata_object->getValue(Iceberg::f_format_version); auto schema_fields = current_schema->getArray(Iceberg::f_fields); @@ -1029,19 +963,12 @@ bool MetadataGenerator::generateModifyColumnMetadata(const String & column_name, if (current_field->getValue(Iceberg::f_name) != column_name) continue; -<<<<<<< HEAD if (current_field->getValue(Iceberg::f_required) == new_type.second && icebergTypesEqualIgnoringIds(current_field->get(Iceberg::f_type), new_type.first)) { Iceberg::IcebergSchemaProcessor schema_processor( context, context->getSettingsRef()[Setting::allow_experimental_geo_types_in_iceberg]); auto current_ch_type = schema_processor.getClickHouseFieldType(current_field, context); -======= - auto old_type = Iceberg::deepCopy(current_field); - current_field->set(Iceberg::f_type, new_type.first); - if (!current_field->getValue(Iceberg::f_required) && !type->isNullable()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Iceberg spec doesn't allow change type from nullable to non-nullable {}", type->getPrettyName()); ->>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) if (!current_ch_type->equals(*type)) throw Exception( @@ -1086,7 +1013,7 @@ bool MetadataGenerator::generateModifyColumnMetadata(const String & column_name, return false; const auto next_schema_id = getNextSchemaId(metadata_object); - current_schema = deepCopy(current_schema); + current_schema = Iceberg::deepCopy(current_schema); schema_fields = current_schema->getArray(Iceberg::f_fields); auto target_field = schema_fields->getObject(target_index); @@ -1137,26 +1064,7 @@ bool MetadataGenerator::generateModifyColumnMetadata(const String & column_name, void MetadataGenerator::generateRenameColumnMetadata(const String & column_name, const String & new_column_name) { -<<<<<<< HEAD - auto current_schema = deepCopy(getCurrentSchema()); -======= - auto current_schema_id = metadata_object->getValue(Iceberg::f_current_schema_id); - - Poco::JSON::Object::Ptr current_schema; - auto schemas = metadata_object->getArray(Iceberg::f_schemas); - for (UInt32 i = 0; i < schemas->size(); ++i) - { - if (schemas->getObject(i)->getValue(Iceberg::f_schema_id) == current_schema_id) - { - current_schema = schemas->getObject(i); - break; - } - } - - if (!current_schema) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Not found schema with id {}", current_schema_id); - current_schema = Iceberg::deepCopy(current_schema); ->>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) + auto current_schema = Iceberg::deepCopy(getCurrentSchema()); auto schema_fields = current_schema->getArray(Iceberg::f_fields); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp index cf064d55d0ea7..28b7f5bdd329a 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp @@ -790,12 +790,8 @@ void mutate( filename_generator.setVersion(last_version + 1); filename_generator.setCompressionMethod(compression_method); -<<<<<<< HEAD - auto metadata = getMetadataJSONObject(metadata_path, object_storage, persistent_table_components.metadata_cache, context, log, compression_method, persistent_table_components.table_uuid); - auto previous_metadata_path = persistent_table_components.path_resolver.reverseResolve(metadata_path); -======= auto metadata = deepCopy(getMetadataJSONObject(metadata_path, object_storage, persistent_table_components.metadata_cache, context, log, compression_method, persistent_table_components.table_uuid)); ->>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) + auto previous_metadata_path = persistent_table_components.path_resolver.reverseResolve(metadata_path); /// Iceberg v3 writers must not add new position-delete files; row-level deletes require /// deletion vectors. Fail closed before any object writes until ClickHouse can write DVs. diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp index b62476d81faa4..b22a389b7ef5f 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp @@ -591,9 +591,6 @@ String readMetadataJSONString( LoggerPtr log, CompressionMethod compression_method) { - ProfileEvents::increment(ProfileEvents::IcebergMetadataJSONReads); - ProfileEventTimeIncrement watch(ProfileEvents::IcebergMetadataJSONReadMicroseconds); - ObjectInfo object_info(metadata_file_path); auto read_settings = local_context->getReadSettings(); @@ -619,9 +616,8 @@ String readMetadataJSONString( Poco::JSON::Object::Ptr parseMetadataJSONString(const String & metadata_json_str) { - ProfileEvents::increment(ProfileEvents::IcebergMetadataJSONParses); - ProfileEvents::increment(ProfileEvents::IcebergMetadataJSONParsedBytes, metadata_json_str.size()); - ProfileEventTimeIncrement watch(ProfileEvents::IcebergMetadataJSONParseMicroseconds); + ProfileEvents::increment(ProfileEvents::IcebergJsonFileParsing); + ProfileEventTimeIncrement watch(ProfileEvents::IcebergJsonFileParsingMicroseconds); Poco::JSON::Parser parser; /// For some reason base/base/JSON.h can not parse this json file Poco::Dynamic::Var json = parser.parse(metadata_json_str); @@ -681,44 +677,6 @@ Poco::JSON::Object::Ptr getMetadataJSONObject( { auto read_fn = [&]() { -<<<<<<< HEAD - ObjectInfo object_info(metadata_file_path); - - auto read_settings = local_context->getReadSettings(); - /// Do not utilize filesystem cache if more precise cache enabled - if (metadata_cache) - read_settings.enable_filesystem_cache = false; - - auto source_buf = createReadBuffer(object_info.relative_path_with_metadata, object_storage, local_context, log, read_settings); - - std::unique_ptr buf; - if (compression_method != CompressionMethod::None) - /// Iceberg metadata snappy is always the Hadoop block format (`SnappyMode::Basic`); the - /// write path in `writeMessageToFile` pins the same mode. The wire format cannot be - /// inferred from the `.snappy.metadata.json` suffix, so both sides must agree statically. - buf = wrapReadBufferWithCompressionMethod(std::move(source_buf), compression_method, /*zstd_window_log_max=*/ 0, SnappyMode::Basic); - else - buf = std::move(source_buf); - - String json_str; - readJSONObjectPossiblyInvalid(json_str, *buf); - return json_str; - }; - - ProfileEvents::increment(ProfileEvents::IcebergJsonFileParsing); - ProfileEventTimeIncrement watch(ProfileEvents::IcebergJsonFileParsingMicroseconds); - - String metadata_json_str; - if (metadata_cache && table_uuid.has_value()) - metadata_json_str = metadata_cache->getOrSetTableMetadata( - IcebergMetadataFilesCache::getKey(*table_uuid, metadata_file_path), create_fn); - else - metadata_json_str = create_fn(); - - Poco::JSON::Parser parser; /// For some reason base/base/JSON.h can not parse this json file - Poco::Dynamic::Var json = parser.parse(metadata_json_str); - return json.extract(); -======= return readMetadataJSONString(metadata_file_path, object_storage, metadata_cache != nullptr, local_context, log, compression_method); }; @@ -732,7 +690,6 @@ Poco::JSON::Object::Ptr getMetadataJSONObject( auto metadata_json_str = read_fn(); return std::make_shared(parseMetadataJSONString(metadata_json_str), metadata_json_str.size()); }); ->>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) } /// Iceberg stores a decimal as its unscaled value in two's-complement big-endian form, in the diff --git a/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp b/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp index 3cbcc789c7b59..441c9cf5e4248 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp @@ -313,16 +313,13 @@ void StorageObjectStorageCluster::prepareForDrop(ContextPtr query_context) std::optional StorageObjectStorageCluster::totalRows(ContextPtr query_context) const { -<<<<<<< HEAD - if (pure_storage) - return pure_storage->totalRows(query_context); -======= return totalRowsInSnapshot(nullptr, query_context); } std::optional StorageObjectStorageCluster::totalRowsInSnapshot(const StorageSnapshotPtr & storage_snapshot, ContextPtr query_context) const { ->>>>>>> c1146e5434f9 (Merge pull request #126267 from ClickHouse/iceberg-cache-parsed-metadata-json) + if (pure_storage) + return pure_storage->totalRowsInSnapshot(storage_snapshot, query_context); configuration->lazyInitializeIfNeeded( object_storage, query_context);