From cafb68c76f6283ec5f92d1653836453703087f3c Mon Sep 17 00:00:00 2001 From: Nikita Kazeev Date: Thu, 19 Nov 2026 20:47:54 +0800 Subject: [PATCH 1/3] fix: batch parent queries during recursive listings --- pydrive2/fs/spec.py | 18 ++++-- pydrive2/test/test_fs_listing.py | 102 +++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 6 deletions(-) create mode 100644 pydrive2/test/test_fs_listing.py diff --git a/pydrive2/fs/spec.py b/pydrive2/fs/spec.py index b39306e..1817220 100644 --- a/pydrive2/fs/spec.py +++ b/pydrive2/fs/spec.py @@ -7,7 +7,7 @@ import threading from collections import defaultdict from contextlib import contextmanager -from itertools import chain +from itertools import chain, islice from fsspec.spec import AbstractFileSystem from funcy import cached_property, retry, wrap_prop, wrap_with @@ -311,11 +311,17 @@ def _gdrive_list(self, query): return chain.from_iterable(iter(get_list, None)) def _gdrive_list_ids(self, query_ids): - query = " or ".join( - f"'{query_id}' in parents" for query_id in query_ids - ) - query = f"({query}) and trashed=false" - return self._gdrive_list(query) + # Large parent queries can exceed Drive's query complexity limit. + query_ids = iter(query_ids) + while True: + batch = list(islice(query_ids, 50)) + if not batch: + return + query = " or ".join( + f"'{query_id}' in parents" for query_id in batch + ) + query = f"({query}) and trashed=false" + yield from self._gdrive_list(query) def _get_remote_item_ids( self, parent_ids, parent_path, title, use_cache=True diff --git a/pydrive2/test/test_fs_listing.py b/pydrive2/test/test_fs_listing.py new file mode 100644 index 0000000..9cc9e2b --- /dev/null +++ b/pydrive2/test/test_fs_listing.py @@ -0,0 +1,102 @@ +import re +from unittest.mock import Mock + +import pytest + +from pydrive2.fs import GDriveFileSystem +from pydrive2.fs.spec import FOLDER_MIME_TYPE + + +@pytest.fixture +def listing_fs(): + return GDriveFileSystem( + "root", google_auth=Mock(), skip_instance_cache=True + ) + + +@pytest.mark.parametrize("count", [0, 1, 50, 51, 100, 251]) +@pytest.mark.parametrize("container", [list, dict.fromkeys, iter]) +def test_list_ids_batches(listing_fs, monkeypatch, count, container): + ids = [f"parent-{index}" for index in range(count)] + listed = [] + + def list_query(query): + parents = re.findall(r"'([^']+)' in parents", query) + # Model an API that rejects overly complex parent queries. + assert 0 < len(parents) <= 50 + assert query.endswith(") and trashed=false") + listed.extend(parents) + return iter(parents) + + monkeypatch.setattr(listing_fs, "_gdrive_list", list_query) + + assert list(listing_fs._gdrive_list_ids(container(ids))) == ids + assert listed == ids + + +def test_list_ids_lazy(listing_fs, monkeypatch): + consumed = [] + + def parents(): + for index in range(51): + consumed.append(index) + yield str(index) + + list_query = Mock(side_effect=[iter(["page-1", "page-2"]), iter(["last"])]) + monkeypatch.setattr(listing_fs, "_gdrive_list", list_query) + result = listing_fs._gdrive_list_ids(parents()) + assert consumed == [] + list_query.assert_not_called() + + assert next(result) == "page-1" + assert len(consumed) == 50 + assert next(result) == "page-2" + assert list_query.call_count == 1 + assert list(result) == ["last"] + assert len(consumed) == 51 + assert list_query.call_count == 2 + + +@pytest.mark.parametrize("detail", [False, True]) +def test_find_wide_directory(listing_fs, monkeypatch, detail): + folders = [f"folder-{index}" for index in range(251)] + + def list_query(query): + parents = re.findall(r"'([^']+)' in parents", query) + assert 0 < len(parents) <= 50 + if parents == ["root"]: + return iter( + { + "id": folder, + "title": folder, + "mimeType": FOLDER_MIME_TYPE, + "parents": [{"id": "root"}], + } + for folder in folders + ) + return iter( + { + "title": "data", + "mimeType": "application/octet-stream", + "parents": [{"id": parent}], + "fileSize": "7", + "md5Checksum": "checksum", + } + for parent in parents + ) + + monkeypatch.setattr(listing_fs, "_gdrive_list", list_query) + files = listing_fs.find("root", detail=detail) + expected = [f"root/{folder}/data" for folder in folders] + assert list(files) == expected + if detail: + assert all( + item + == { + "name": path, + "type": "file", + "size": 7, + "checksum": "checksum", + } + for path, item in files.items() + ) From 87c17bb1e2511d97db7a1919f24b30da83a03f18 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sat, 21 Nov 2026 11:21:02 -0800 Subject: [PATCH 2/3] Batch path lookups and query each Drive parent only once --- pydrive2/fs/spec.py | 46 ++++++++++++++------------- pydrive2/test/test_fs_listing.py | 54 ++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 21 deletions(-) diff --git a/pydrive2/fs/spec.py b/pydrive2/fs/spec.py index 3a4de97..d200fff 100644 --- a/pydrive2/fs/spec.py +++ b/pydrive2/fs/spec.py @@ -6,12 +6,19 @@ import threading from collections import defaultdict from contextlib import contextmanager -from itertools import chain, islice +from itertools import chain from typing import TypedDict import appdirs from fsspec.spec import AbstractFileSystem -from funcy import cached_property, retry, wrap_prop, wrap_with +from funcy import ( + cached_property, + chunks, + distinct, + retry, + wrap_prop, + wrap_with, +) from tqdm.utils import CallbackIOWrapper from pydrive2.auth import GoogleAuth, InvalidCredentialsError @@ -21,6 +28,7 @@ logger = logging.getLogger(__name__) FOLDER_MIME_TYPE = "application/vnd.google-apps.folder" +_PARENT_QUERY_BATCH_SIZE = 50 COMMON_SETTINGS = { "get_refresh_token": True, @@ -40,6 +48,11 @@ class GDriveAuthError(Exception): pass +def _quote_query_value(value): + escaped = value.replace("\\", "\\\\").replace("'", "\\'") + return f"'{escaped}'" + + def _gdrive_retry(func): def should_retry(exc): from pydrive2.files import ApiRequestError @@ -317,33 +330,24 @@ def _gdrive_list(self, query): # Fetch pages until None is received, lazily flatten the thing. return chain.from_iterable(iter(get_list, None)) - def _gdrive_list_ids(self, query_ids): - # Large parent queries can exceed Drive's query complexity limit. - query_ids = iter(query_ids) - while True: - batch = list(islice(query_ids, 50)) - if not batch: - return - query = " or ".join( - f"'{query_id}' in parents" for query_id in batch + def _gdrive_list_ids(self, query_ids, *, title=None): + filters = "trashed=false" + if title is not None: + filters += f" and title={_quote_query_value(title)}" + # Drive can reject large parent queries. Query each parent only once. + for batch in chunks(_PARENT_QUERY_BATCH_SIZE, distinct(query_ids)): + parents = " or ".join( + f"{_quote_query_value(parent)} in parents" for parent in batch ) - query = f"({query}) and trashed=false" - yield from self._gdrive_list(query) + yield from self._gdrive_list(f"({parents}) and {filters}") def _get_remote_item_ids( self, parent_ids, parent_path, title, use_cache=True ): if not parent_ids: return None - query = "trashed=false and ({})".format( - " or ".join( - f"'{parent_id}' in parents" for parent_id in parent_ids - ) - ) - query += " and title='{}'".format(title.replace("'", "\\'")) - res = [] - for item in self._gdrive_list(query): + for item in self._gdrive_list_ids(parent_ids, title=title): # GDrive list API is case insensitive, we need to compare # all results and pick the ones with the right title if item["title"] == title: diff --git a/pydrive2/test/test_fs_listing.py b/pydrive2/test/test_fs_listing.py index 9cc9e2b..49e7848 100644 --- a/pydrive2/test/test_fs_listing.py +++ b/pydrive2/test/test_fs_listing.py @@ -57,6 +57,60 @@ def parents(): assert list_query.call_count == 2 +def test_ls_repeated_cached_parent_across_batches(listing_fs, monkeypatch): + parents = [f"id-{i}" for i in range(51)] + listing_fs._cache_path_id("duplicate", *parents, parents[0]) + batches = [] + + def list_file(params): + batch = re.findall(r"'([^']+)' in parents", params["q"]) + batches.append(batch) + # Distinct children with the same title must all remain in the listing. + children = [ + { + "id": f"child-{parent}", + "title": "file", + "mimeType": "text/plain", + } + for parent in dict.fromkeys(batch) + ] + return iter([children[:1], [], children[1:]]) + + monkeypatch.setattr(listing_fs.client, "ListFile", list_file) + assert listing_fs.ls("root/duplicate") == ["root/duplicate/file"] * 51 + assert batches == [parents[:50], parents[50:]] + + +def test_path_lookup_batches_duplicate_folder_names(listing_fs, monkeypatch): + parents = [f"id-{i}" for i in range(51)] + listing_fs._cache_path_id("duplicate", *parents) + title = r"File's\name" + batches = [] + + def list_file(params): + query = params["q"] + batch = re.findall(r"'([^']+)' in parents", query) + assert 0 < len(batch) <= 50 + assert r"title='File\'s\\name'" in query + assert "trashed=false" in query + batches.append(batch) + if parents[-1] in batch: + item_id, item_title = "found", title + else: + item_id, item_title = "wrong-case", title.lower() + item = { + "id": item_id, + "title": item_title, + "mimeType": FOLDER_MIME_TYPE, + } + return iter([[item]]) + + monkeypatch.setattr(listing_fs.client, "ListFile", list_file) + assert listing_fs.exists(f"root/duplicate/{title}") + assert listing_fs._ids_cache["dirs"][f"duplicate/{title}"] == ["found"] + assert batches == [parents[:50], parents[50:]] + + @pytest.mark.parametrize("detail", [False, True]) def test_find_wide_directory(listing_fs, monkeypatch, detail): folders = [f"folder-{index}" for index in range(251)] From 837b0d73e5ebac6b982f57c044b5e2ce987fb559 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sat, 21 Nov 2026 11:41:34 -0800 Subject: [PATCH 3/3] Snapshot cached parent IDs before batched Drive lookups --- pydrive2/fs/spec.py | 3 ++- pydrive2/test/test_fs_listing.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/pydrive2/fs/spec.py b/pydrive2/fs/spec.py index d200fff..880dea1 100644 --- a/pydrive2/fs/spec.py +++ b/pydrive2/fs/spec.py @@ -364,7 +364,8 @@ def _get_cached_item_ids(self, path, use_cache): if not path: return [self.root] if use_cache: - return self._ids_cache["dirs"].get(path, []) + # Another caller can add parents while a batched lookup is running. + return self._ids_cache["dirs"].get(path, []).copy() return [] def _path_to_item_ids(self, path, create=False, use_cache=True): diff --git a/pydrive2/test/test_fs_listing.py b/pydrive2/test/test_fs_listing.py index 49e7848..9ab5dd3 100644 --- a/pydrive2/test/test_fs_listing.py +++ b/pydrive2/test/test_fs_listing.py @@ -81,6 +81,23 @@ def list_file(params): assert batches == [parents[:50], parents[50:]] +def test_ls_uses_parents_cached_at_start(listing_fs, monkeypatch): + parents = [f"id-{i}" for i in range(51)] + listing_fs._cache_path_id("dir", *parents) + + def list_query(query): + if "'id-0' in parents" in query: + # Simulate another caller caching a parent during the first batch. + listing_fs._cache_path_id("dir", "late-parent") + if "'late-parent' in parents" in query: + return [{"title": "late-file", "mimeType": "text/plain"}] + return [] + + monkeypatch.setattr(listing_fs, "_gdrive_list", list_query) + assert listing_fs.ls("root/dir") == [] + assert listing_fs.ls("root/dir") == ["root/dir/late-file"] + + def test_path_lookup_batches_duplicate_folder_names(listing_fs, monkeypatch): parents = [f"id-{i}" for i in range(51)] listing_fs._cache_path_id("duplicate", *parents)