From 42e76b097b191071bff48a0e9ea6482fd48ff178 Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Tue, 17 Nov 2026 04:50:29 +0000 Subject: [PATCH 1/2] feat(bigquery): add bigquery_upgrade_manifest_biglake_table sample to upgrade a manifest external table to BigLake with automatic metadata caching Problem: The BigQuery documentation (https://cloud.google.com/bigquery/docs/query-open-table-format-using-manifest-files) needs a standalone, tested Python code sample demonstrating how to upgrade a manifest external table to BigLake with automatic metadata caching (region tag `bigquery_upgrade_manifest_biglake_table`). Solution: - Add `upgrade_manifest_biglake_table(project_id: str, dataset_id: str, table_id: str, connection_id: str, manifest_uri: str) -> None` in `bigquery/biglake/bigquery_upgrade_manifest_biglake_table.py` (Upgrade a manifest-backed external table to a BigLake table with AUTOMATIC metadata caching and connection binding.) - Add unit tests (`bigquery/biglake/bigquery_upgrade_manifest_biglake_table_test.py`) covering both the success path and API error handling (`GoogleAPICallError`) --- ...bigquery_upgrade_manifest_biglake_table.py | 80 +++++++++++++++++++ ...ery_upgrade_manifest_biglake_table_test.py | 75 +++++++++++++++++ 2 files changed, 155 insertions(+) create mode 100644 bigquery/biglake/bigquery_upgrade_manifest_biglake_table.py create mode 100644 bigquery/biglake/tests/bigquery_upgrade_manifest_biglake_table_test.py diff --git a/bigquery/biglake/bigquery_upgrade_manifest_biglake_table.py b/bigquery/biglake/bigquery_upgrade_manifest_biglake_table.py new file mode 100644 index 0000000000..d374dbc834 --- /dev/null +++ b/bigquery/biglake/bigquery_upgrade_manifest_biglake_table.py @@ -0,0 +1,80 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START bigquery_upgrade_manifest_biglake_table] +from google.cloud import bigquery + + +def upgrade_manifest_biglake_table( + project_id: str, + dataset_id: str, + table_id: str, + connection_id: str, + manifest_uri: str, +) -> None: + """Upgrades a manifest external table to a BigLake table with metadata caching. + + Args: + project_id: Google Cloud project ID (for example, 'example-project'). + dataset_id: BigQuery dataset ID containing the external table + (for example, 'example_dataset'). + table_id: BigQuery external table ID to upgrade + (for example, 'example_manifest_table'). + connection_id: Fully qualified BigQuery Cloud Resource connection ID + (for example, 'example-project.us.example_biglake_conn'). + manifest_uri: Cloud Storage URI of the SymLinkTextInputFormat manifest file + (for example, 'gs://example-bucket/manifests/symlink.manifest'). + """ + client = bigquery.Client(project=project_id) + table_ref = f"{project_id}.{dataset_id}.{table_id}" + # Upgrading the manifest external table with a Cloud Resource connection and AUTOMATIC metadata + # caching accelerates open-table-format reads and enforces BigLake governance. + ddl_sql = f""" + CREATE OR REPLACE EXTERNAL TABLE `{table_ref}` + WITH CONNECTION `{connection_id}` + OPTIONS ( + format = 'PARQUET', + uris = ['{manifest_uri}'], + file_set_spec_type = 'NEW_LINE_DELIMITED_MANIFEST', + max_staleness = INTERVAL 1 HOUR, + metadata_cache_mode = 'AUTOMATIC' + ); + """ + select_sql = f""" + SELECT + id, + label, + score + FROM `{table_ref}` + ORDER BY id ASC + LIMIT 10; + """ + + print( + f"Upgrading manifest external table to BigLake with metadata caching: {table_ref}" + ) + client.query(ddl_sql).result() + + query_job = client.query(select_sql) + rows = list(query_job.result()) + + print(f"Resource ID: {table_ref}") + print(f"Connection ID: {connection_id}") + print(f"Job ID: {query_job.job_id}") + print(f"Manifest URI: {manifest_uri}") + print("Metadata Cache Mode: AUTOMATIC") + print(f"Rows Returned: {len(rows)}") + + +# [END bigquery_upgrade_manifest_biglake_table] diff --git a/bigquery/biglake/tests/bigquery_upgrade_manifest_biglake_table_test.py b/bigquery/biglake/tests/bigquery_upgrade_manifest_biglake_table_test.py new file mode 100644 index 0000000000..11c252aedc --- /dev/null +++ b/bigquery/biglake/tests/bigquery_upgrade_manifest_biglake_table_test.py @@ -0,0 +1,75 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import mock + +from google.api_core import exceptions +import pytest + +from .. import bigquery_upgrade_manifest_biglake_table + + +def test_upgrade_manifest_biglake_table_success( + capsys: pytest.CaptureFixture[str], +) -> None: + with mock.patch.object( + bigquery_upgrade_manifest_biglake_table.bigquery, "Client" + ) as mock_client_cls: + mock_client = mock_client_cls.return_value + mock_ddl_job = mock.MagicMock() + mock_ddl_job.result.return_value = [] + mock_select_job = mock.MagicMock() + mock_select_job.job_id = "example-job-123" + mock_select_job.result.return_value = [ + mock.MagicMock(), + mock.MagicMock(), + mock.MagicMock(), + ] + mock_client.query.side_effect = [mock_ddl_job, mock_select_job] + + bigquery_upgrade_manifest_biglake_table.upgrade_manifest_biglake_table( + project_id="example-project", + dataset_id="example_dataset", + table_id="example_manifest_table", + connection_id="example-project.us.example_biglake_conn", + manifest_uri="gs://example-bucket/manifests/symlink.manifest", + ) + + assert mock_client.query.call_count == 2 + captured = capsys.readouterr() + assert ( + "Resource ID: example-project.example_dataset.example_manifest_table" + in captured.out + ) + assert "Metadata Cache Mode: AUTOMATIC" in captured.out + assert "Rows Returned: 3" in captured.out + + +def test_upgrade_manifest_biglake_table_not_found() -> None: + with mock.patch.object( + bigquery_upgrade_manifest_biglake_table.bigquery, "Client" + ) as mock_client_cls: + mock_client = mock_client_cls.return_value + mock_job = mock.MagicMock() + mock_job.result.side_effect = exceptions.NotFound("Connection not found") + mock_client.query.return_value = mock_job + + with pytest.raises(exceptions.GoogleAPICallError): + bigquery_upgrade_manifest_biglake_table.upgrade_manifest_biglake_table( + project_id="example-project", + dataset_id="example_dataset", + table_id="example_manifest_table", + connection_id="example-project.us.example_biglake_conn", + manifest_uri="gs://example-bucket/manifests/symlink.manifest", + ) From c8cc3633c88d9e3ea7a8080bdc289efa2e8eb702 Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Tue, 17 Nov 2026 05:40:07 +0000 Subject: [PATCH 2/2] fix(bigquery): report applied metadata cache settings in upgrade_manifest_biglake_table Problem: Review feedback on this sample pointed out two issues: - It printed "Metadata Cache Mode: AUTOMATIC" as a hardcoded string, so neither the sample nor its test verified that BigQuery applied the setting. - The verification query selected id, label and score, which only exist in our test data, so the sample failed against any other manifest table. Solution: - After the CREATE statement, read the table back with client.get_table() and print the metadataCacheMode from its external data configuration and the table's max_staleness. ExternalConfig has no metadata_cache_mode property in google-cloud-bigquery 3.47.0, so the value is read from the public to_api_repr() output. - Use SELECT * ... LIMIT 10 so the query works with any schema. - The test mocks get_table(), asserts it is called with the table ID, and checks the reported cache mode and max staleness. --- .../bigquery_upgrade_manifest_biglake_table.py | 14 ++++++++------ ...bigquery_upgrade_manifest_biglake_table_test.py | 11 +++++++++++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/bigquery/biglake/bigquery_upgrade_manifest_biglake_table.py b/bigquery/biglake/bigquery_upgrade_manifest_biglake_table.py index d374dbc834..d835bafc21 100644 --- a/bigquery/biglake/bigquery_upgrade_manifest_biglake_table.py +++ b/bigquery/biglake/bigquery_upgrade_manifest_biglake_table.py @@ -52,12 +52,8 @@ def upgrade_manifest_biglake_table( ); """ select_sql = f""" - SELECT - id, - label, - score + SELECT * FROM `{table_ref}` - ORDER BY id ASC LIMIT 10; """ @@ -66,6 +62,11 @@ def upgrade_manifest_biglake_table( ) client.query(ddl_sql).result() + # Read the table back to confirm the metadata caching settings BigQuery applied. + table = client.get_table(table_ref) + external_config = table.external_data_configuration.to_api_repr() + metadata_cache_mode = external_config.get("metadataCacheMode") + query_job = client.query(select_sql) rows = list(query_job.result()) @@ -73,7 +74,8 @@ def upgrade_manifest_biglake_table( print(f"Connection ID: {connection_id}") print(f"Job ID: {query_job.job_id}") print(f"Manifest URI: {manifest_uri}") - print("Metadata Cache Mode: AUTOMATIC") + print(f"Metadata Cache Mode: {metadata_cache_mode}") + print(f"Max Staleness: {table.max_staleness}") print(f"Rows Returned: {len(rows)}") diff --git a/bigquery/biglake/tests/bigquery_upgrade_manifest_biglake_table_test.py b/bigquery/biglake/tests/bigquery_upgrade_manifest_biglake_table_test.py index 11c252aedc..f20a3b50b4 100644 --- a/bigquery/biglake/tests/bigquery_upgrade_manifest_biglake_table_test.py +++ b/bigquery/biglake/tests/bigquery_upgrade_manifest_biglake_table_test.py @@ -37,6 +37,13 @@ def test_upgrade_manifest_biglake_table_success( mock.MagicMock(), ] mock_client.query.side_effect = [mock_ddl_job, mock_select_job] + mock_table = mock.MagicMock() + mock_table.external_data_configuration.to_api_repr.return_value = { + "sourceFormat": "PARQUET", + "metadataCacheMode": "AUTOMATIC", + } + mock_table.max_staleness = "0-0 0 1:0:0" + mock_client.get_table.return_value = mock_table bigquery_upgrade_manifest_biglake_table.upgrade_manifest_biglake_table( project_id="example-project", @@ -47,12 +54,16 @@ def test_upgrade_manifest_biglake_table_success( ) assert mock_client.query.call_count == 2 + mock_client.get_table.assert_called_once_with( + "example-project.example_dataset.example_manifest_table" + ) captured = capsys.readouterr() assert ( "Resource ID: example-project.example_dataset.example_manifest_table" in captured.out ) assert "Metadata Cache Mode: AUTOMATIC" in captured.out + assert "Max Staleness: 0-0 0 1:0:0" in captured.out assert "Rows Returned: 3" in captured.out