Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions bigquery/biglake/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
25 changes: 3 additions & 22 deletions bigquery/biglake/bigquery_create_manifest_biglake_table.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,6 @@
# limitations under the License.

# [START bigquery_create_manifest_biglake_table]
import sys

from google.api_core import exceptions
from google.cloud import bigquery


Expand Down Expand Up @@ -52,25 +49,9 @@ def create_manifest_biglake_table(
);
"""

try:
print(f"Creating manifest-backed BigLake table: {table_ref}")
query_job = client.query(sql)
query_job.result()
except exceptions.NotFound as err:
print(
f"error: dataset '{dataset_id}' or connection '{connection_id}' not found "
f"for project '{project_id}' (manifest '{manifest_uri}'): {err.message}",
file=sys.stderr,
)
return
except exceptions.GoogleAPICallError as err:
print(
f"error: failed to create BigLake table '{table_id}' in dataset '{dataset_id}' "
f"for project '{project_id}' (connection '{connection_id}', "
f"manifest '{manifest_uri}'): {err.message}",
file=sys.stderr,
)
return
print(f"Creating manifest-backed BigLake table: {table_ref}")
query_job = client.query(sql)
query_job.result()

print(f"Resource ID: {table_ref}")
print(f"Connection ID: {connection_id}")
Expand Down
56 changes: 22 additions & 34 deletions bigquery/biglake/bigquery_query_iceberg_table.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,6 @@
# limitations under the License.

# [START bigquery_query_iceberg_table]
import sys

from google.api_core import exceptions
from google.cloud import bigquery


Expand Down Expand Up @@ -43,38 +40,29 @@ def query_iceberg_table(
table_ref = f"{project_id}.{dataset_id}.{table_id}"
# BigQuery Iceberg external tables point to a specific Iceberg metadata JSON snapshot;
# re-running CREATE OR REPLACE EXTERNAL TABLE updates the snapshot URI with schema autodetection.
query = (
f"CREATE OR REPLACE EXTERNAL TABLE `{table_ref}` "
f"WITH CONNECTION `{connection_id}` "
"OPTIONS ("
" format = 'ICEBERG',"
f" uris = ['{metadata_uri}']"
"); "
"SELECT order_id, user_id, status, num_of_item "
f"FROM `{table_ref}` "
"ORDER BY order_id ASC;"
)
ddl_sql = f"""
CREATE OR REPLACE EXTERNAL TABLE `{table_ref}`
WITH CONNECTION `{connection_id}`
OPTIONS (
format = 'ICEBERG',
uris = ['{metadata_uri}']
);
"""
select_sql = f"""
SELECT
order_id,
user_id,
status,
num_of_item
FROM `{table_ref}`
ORDER BY order_id ASC;
"""

print(f"Refreshing Iceberg metadata and querying table: {table_ref}")
client.query(ddl_sql).result()

try:
print(f"Refreshing Iceberg metadata and querying table: {table_ref}")
query_job = client.query(query)
rows = list(query_job.result())
except exceptions.NotFound as err:
print(
f"error: Apache Iceberg table '{table_id}' in dataset '{dataset_id}' "
f"for project '{project_id}' (connection '{connection_id}', "
f"metadata '{metadata_uri}') was not found: {err.message}",
file=sys.stderr,
)
return
except exceptions.GoogleAPICallError as err:
print(
f"error: could not refresh or query Apache Iceberg table '{table_id}' "
f"in dataset '{dataset_id}' for project '{project_id}' "
f"(connection '{connection_id}', metadata '{metadata_uri}'): {err.message}",
file=sys.stderr,
)
return
query_job = client.query(select_sql)
rows = list(query_job.result())

print(f"Resource ID: {table_ref}")
print(f"Job ID: {query_job.job_id}")
Expand Down
26 changes: 4 additions & 22 deletions bigquery/biglake/bigquery_query_manifest_external_table.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,6 @@
# limitations under the License.

# [START bigquery_query_manifest_external_table]
import sys

from google.api_core import exceptions
from google.cloud import bigquery


Expand Down Expand Up @@ -54,25 +51,10 @@ def query_manifest_external_table(
"SELECT id, label, score " f"FROM `{table_ref}` " "ORDER BY id ASC " "LIMIT 10"
)

try:
print(f"Configuring manifest URI and querying external table: {table_ref}")
client.update_table(table, ["external_data_configuration"])
query_job = client.query(query)
rows = list(query_job.result())
except exceptions.NotFound as err:
print(
f"error: manifest external table '{table_id}' in dataset '{dataset_id}' "
f"for project '{project_id}' (manifest '{manifest_uri}') was not found: {err.message}",
file=sys.stderr,
)
return
except exceptions.GoogleAPICallError as err:
print(
f"error: could not query manifest external table '{table_id}' in dataset "
f"'{dataset_id}' for project '{project_id}' (manifest '{manifest_uri}'): {err.message}",
file=sys.stderr,
)
return
print(f"Configuring manifest URI and querying external table: {table_ref}")
client.update_table(table, ["external_data_configuration"])
query_job = client.query(query)
rows = list(query_job.result())

print(f"Resource ID: {table_ref}")
print(f"Job ID: {query_job.job_id}")
Expand Down
13 changes: 13 additions & 0 deletions bigquery/biglake/tests/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,18 @@

from unittest import mock

import bigquery_create_manifest_biglake_table
from google.api_core import exceptions
import pytest

from .. import bigquery_create_manifest_biglake_table


def test_create_manifest_biglake_table_success(
capsys: pytest.CaptureFixture[str],
) -> None:
with mock.patch("google.cloud.bigquery.Client") as mock_client_cls:
with mock.patch.object(
bigquery_create_manifest_biglake_table.bigquery, "Client"
) as mock_client_cls:
mock_client = mock_client_cls.return_value
mock_job = mock.MagicMock()
mock_job.job_id = "example-job-123"
Expand All @@ -47,20 +50,19 @@ def test_create_manifest_biglake_table_success(
assert "Statement Type: CREATE_EXTERNAL_TABLE" in captured.out


def test_create_manifest_biglake_table_not_found(
capsys: pytest.CaptureFixture[str],
) -> None:
with mock.patch("google.cloud.bigquery.Client") as mock_client_cls:
def test_create_manifest_biglake_table_not_found() -> None:
with mock.patch.object(
bigquery_create_manifest_biglake_table.bigquery, "Client"
) as mock_client_cls:
mock_client = mock_client_cls.return_value
mock_client.query.side_effect = exceptions.NotFound("Dataset not found")

bigquery_create_manifest_biglake_table.create_manifest_biglake_table(
project_id="example-project",
dataset_id="example_dataset",
connection_id="example-project.us.example_biglake_conn",
manifest_uri="gs://example-bucket/manifests/symlink.manifest",
)
mock_job = mock.MagicMock()
mock_job.result.side_effect = exceptions.NotFound("Dataset not found")
mock_client.query.return_value = mock_job

captured = capsys.readouterr()
assert "error:" in captured.err
assert "example_dataset" in captured.err
with pytest.raises(exceptions.GoogleAPICallError):
bigquery_create_manifest_biglake_table.create_manifest_biglake_table(
project_id="example-project",
dataset_id="example_dataset",
connection_id="example-project.us.example_biglake_conn",
manifest_uri="gs://example-bucket/manifests/symlink.manifest",
)
Original file line number Diff line number Diff line change
Expand Up @@ -14,21 +14,24 @@

from unittest import mock

import bigquery_query_iceberg_table
from google.api_core import exceptions
import pytest

from .. import bigquery_query_iceberg_table


def test_query_iceberg_table_success(
monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str],
) -> None:
mock_client = mock.MagicMock()
mock_job = mock.MagicMock()
mock_job.job_id = "example-job-123"
mock_ddl_job = mock.MagicMock()
mock_ddl_job.result.return_value = []
mock_select_job = mock.MagicMock()
mock_select_job.job_id = "example-job-123"
mock_row = mock.MagicMock()
mock_job.result.return_value = [mock_row]
mock_client.query.return_value = mock_job
mock_select_job.result.return_value = [mock_row]
mock_client.query.side_effect = [mock_ddl_job, mock_select_job]
monkeypatch.setattr(
bigquery_query_iceberg_table.bigquery,
"Client",
Expand All @@ -46,29 +49,27 @@ def test_query_iceberg_table_success(
captured = capsys.readouterr()
assert "Resource ID:" in captured.out
assert "example-project" in captured.out
assert mock_client.query.called
assert mock_client.query.call_count == 2


def test_query_iceberg_table_error(
monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str],
) -> None:
mock_client = mock.MagicMock()
mock_client.query.side_effect = exceptions.NotFound("resource error")
mock_job = mock.MagicMock()
mock_job.result.side_effect = exceptions.NotFound("resource error")
mock_client.query.return_value = mock_job
monkeypatch.setattr(
bigquery_query_iceberg_table.bigquery,
"Client",
lambda project=None: mock_client,
)

bigquery_query_iceberg_table.query_iceberg_table(
project_id="example-project",
dataset_id="example_dataset",
table_id="example_iceberg_table",
connection_id="example-project.us.example_iceberg_conn",
metadata_uri="gs://example-bucket/warehouse/orders_ns/customer_orders/metadata/v2.metadata.json",
)

captured = capsys.readouterr()
assert "error:" in captured.err
assert "example-project" in captured.err
with pytest.raises(exceptions.GoogleAPICallError):
bigquery_query_iceberg_table.query_iceberg_table(
project_id="example-project",
dataset_id="example_dataset",
table_id="example_iceberg_table",
connection_id="example-project.us.example_iceberg_conn",
metadata_uri="gs://example-bucket/warehouse/orders_ns/customer_orders/metadata/v2.metadata.json",
)
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,11 @@

from unittest import mock

import bigquery_query_manifest_external_table
from google.api_core import exceptions
import pytest

from .. import bigquery_query_manifest_external_table


def test_query_manifest_external_table_success(
monkeypatch: pytest.MonkeyPatch,
Expand All @@ -26,7 +27,6 @@ def test_query_manifest_external_table_success(
mock_client = mock.MagicMock()
mock_job = mock.MagicMock()
mock_job.job_id = "example-job-123"
mock_job.statement_type = "SCRIPT"
mock_row = mock.MagicMock()
mock_job.result.return_value = [mock_row]
mock_client.query.return_value = mock_job
Expand All @@ -51,23 +51,21 @@ def test_query_manifest_external_table_success(

def test_query_manifest_external_table_error(
monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str],
) -> None:
mock_client = mock.MagicMock()
mock_client.query.side_effect = exceptions.NotFound("resource error")
mock_job = mock.MagicMock()
mock_job.result.side_effect = exceptions.NotFound("resource error")
mock_client.query.return_value = mock_job
monkeypatch.setattr(
bigquery_query_manifest_external_table.bigquery,
"Client",
lambda project=None: mock_client,
)

bigquery_query_manifest_external_table.query_manifest_external_table(
project_id="example-project",
dataset_id="example_dataset",
table_id="example_manifest_table",
manifest_uri="gs://example-bucket/manifests/symlink.manifest",
)

captured = capsys.readouterr()
assert "error:" in captured.err
assert "example-project" in captured.err
with pytest.raises(exceptions.GoogleAPICallError):
bigquery_query_manifest_external_table.query_manifest_external_table(
project_id="example-project",
dataset_id="example_dataset",
table_id="example_manifest_table",
manifest_uri="gs://example-bucket/manifests/symlink.manifest",
)
13 changes: 13 additions & 0 deletions bigquery/data_profile_scan/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Loading
Loading