Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 82 additions & 0 deletions bigquery/biglake/bigquery_upgrade_manifest_biglake_table.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# [START bigquery_upgrade_manifest_biglake_table]
from google.cloud import bigquery


def upgrade_manifest_biglake_table(
project_id: str,
dataset_id: str,
table_id: str,
connection_id: str,
manifest_uri: str,
) -> None:
"""Upgrades a manifest external table to a BigLake table with metadata caching.

Args:
project_id: Google Cloud project ID (for example, 'example-project').
dataset_id: BigQuery dataset ID containing the external table
(for example, 'example_dataset').
table_id: BigQuery external table ID to upgrade
(for example, 'example_manifest_table').
connection_id: Fully qualified BigQuery Cloud Resource connection ID
(for example, 'example-project.us.example_biglake_conn').
manifest_uri: Cloud Storage URI of the SymLinkTextInputFormat manifest file
(for example, 'gs://example-bucket/manifests/symlink.manifest').
"""
client = bigquery.Client(project=project_id)
table_ref = f"{project_id}.{dataset_id}.{table_id}"
# Upgrading the manifest external table with a Cloud Resource connection and AUTOMATIC metadata
# caching accelerates open-table-format reads and enforces BigLake governance.
ddl_sql = f"""
CREATE OR REPLACE EXTERNAL TABLE `{table_ref}`
WITH CONNECTION `{connection_id}`
OPTIONS (
format = 'PARQUET',
uris = ['{manifest_uri}'],
file_set_spec_type = 'NEW_LINE_DELIMITED_MANIFEST',
max_staleness = INTERVAL 1 HOUR,
metadata_cache_mode = 'AUTOMATIC'
);
"""
select_sql = f"""
SELECT *
FROM `{table_ref}`
LIMIT 10;
"""

print(
f"Upgrading manifest external table to BigLake with metadata caching: {table_ref}"
)
client.query(ddl_sql).result()

# Read the table back to confirm the metadata caching settings BigQuery applied.
table = client.get_table(table_ref)
external_config = table.external_data_configuration.to_api_repr()
metadata_cache_mode = external_config.get("metadataCacheMode")

query_job = client.query(select_sql)
rows = list(query_job.result())

print(f"Resource ID: {table_ref}")
print(f"Connection ID: {connection_id}")
print(f"Job ID: {query_job.job_id}")
print(f"Manifest URI: {manifest_uri}")
print(f"Metadata Cache Mode: {metadata_cache_mode}")
print(f"Max Staleness: {table.max_staleness}")
print(f"Rows Returned: {len(rows)}")


# [END bigquery_upgrade_manifest_biglake_table]
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from unittest import mock

from google.api_core import exceptions
import pytest

from .. import bigquery_upgrade_manifest_biglake_table


def test_upgrade_manifest_biglake_table_success(
capsys: pytest.CaptureFixture[str],
) -> None:
with mock.patch.object(
bigquery_upgrade_manifest_biglake_table.bigquery, "Client"
) as mock_client_cls:
mock_client = mock_client_cls.return_value
mock_ddl_job = mock.MagicMock()
mock_ddl_job.result.return_value = []
mock_select_job = mock.MagicMock()
mock_select_job.job_id = "example-job-123"
mock_select_job.result.return_value = [
mock.MagicMock(),
mock.MagicMock(),
mock.MagicMock(),
]
mock_client.query.side_effect = [mock_ddl_job, mock_select_job]
mock_table = mock.MagicMock()
mock_table.external_data_configuration.to_api_repr.return_value = {
"sourceFormat": "PARQUET",
"metadataCacheMode": "AUTOMATIC",
}
mock_table.max_staleness = "0-0 0 1:0:0"
mock_client.get_table.return_value = mock_table

bigquery_upgrade_manifest_biglake_table.upgrade_manifest_biglake_table(
project_id="example-project",
dataset_id="example_dataset",
table_id="example_manifest_table",
connection_id="example-project.us.example_biglake_conn",
manifest_uri="gs://example-bucket/manifests/symlink.manifest",
)

assert mock_client.query.call_count == 2
mock_client.get_table.assert_called_once_with(
"example-project.example_dataset.example_manifest_table"
)
captured = capsys.readouterr()
assert (
"Resource ID: example-project.example_dataset.example_manifest_table"
in captured.out
)
assert "Metadata Cache Mode: AUTOMATIC" in captured.out
assert "Max Staleness: 0-0 0 1:0:0" in captured.out
assert "Rows Returned: 3" in captured.out


def test_upgrade_manifest_biglake_table_not_found() -> None:
with mock.patch.object(
bigquery_upgrade_manifest_biglake_table.bigquery, "Client"
) as mock_client_cls:
mock_client = mock_client_cls.return_value
mock_job = mock.MagicMock()
mock_job.result.side_effect = exceptions.NotFound("Connection not found")
mock_client.query.return_value = mock_job

with pytest.raises(exceptions.GoogleAPICallError):
bigquery_upgrade_manifest_biglake_table.upgrade_manifest_biglake_table(
project_id="example-project",
dataset_id="example_dataset",
table_id="example_manifest_table",
connection_id="example-project.us.example_biglake_conn",
manifest_uri="gs://example-bucket/manifests/symlink.manifest",
)
Loading