From 90db421abcb1e2c4a8c574f5c3cf243b3410dff4 Mon Sep 17 00:00:00 2001 From: aryankalantri Date: Sun, 22 Nov 2026 02:15:47 +0530 Subject: [PATCH 1/6] feat(fs): register gdrive protocol with fsspec (#354) --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 71886cc..60dfa9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,9 @@ typing = ["mypy==2.4.1", "types-httplib2", "types-PyYAML"] hook-dirs = "pydrive2.__pyinstaller:get_hook_dirs" tests = "pydrive2.__pyinstaller:get_PyInstaller_tests" +[project.entry-points.fsspec] +gdrive = "pydrive2.fs:GDriveFileSystem" + [tool.setuptools] packages=[ "pydrive2", From 2d8994918dcedab44e35386e2f2ad24bc3707a7b Mon Sep 17 00:00:00 2001 From: aryankalantri Date: Sun, 22 Nov 2026 02:16:36 +0530 Subject: [PATCH 2/6] feat(fs): register gdrive protocol with fsspec (#354) --- pydrive2/fs/spec.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pydrive2/fs/spec.py b/pydrive2/fs/spec.py index 880dea1..fa83ef0 100644 --- a/pydrive2/fs/spec.py +++ b/pydrive2/fs/spec.py @@ -177,9 +177,11 @@ def _service_auth( class GDriveFileSystem(AbstractFileSystem): """Access to gdrive as an fsspec filesystem""" + protocol = "gdrive" + def __init__( self, - path, + path="root", google_auth=None, client_id=None, client_secret=None, @@ -194,7 +196,8 @@ def __init__( ): """Create an instance of GDriveFileSystem. - :param path: gdrive path. + :param path: gdrive path. Defaults to ``"root"`` (the Drive + root), so ``fsspec`` can build the filesystem without one. :type path: str. :param google_auth: Authenticated GoogleAuth instance. :type google_auth: GoogleAuth. From b2e7561a6779378d8f98253d093470c61c11cf6f Mon Sep 17 00:00:00 2001 From: aryankalantri Date: Sun, 22 Nov 2026 02:18:23 +0530 Subject: [PATCH 3/6] feat(fs): register gdrive protocol with fsspec (#354) --- docs/fsspec.rst | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/fsspec.rst b/docs/fsspec.rst index 516abeb..f159270 100644 --- a/docs/fsspec.rst +++ b/docs/fsspec.rst @@ -11,6 +11,30 @@ Installation pip install 'pydrive2[fsspec]' +Once installed, ``fsspec`` finds the filesystem under the ``gdrive`` +protocol, so files can be opened without importing PyDrive2 first: + +.. code-block:: python + + import fsspec + + fs = fsspec.filesystem( + "gdrive", + "root", + client_id="my_client_id", +client_secret= + ) + + with fsspec.open( + "gdrive://root/path/to/file.txt", + client_id="my_client_id", +client_secret= + ) as f: + print(f.read()) + +Credentials are still required: without them the filesystem raises +``ValueError`` telling you which credential to supply. + Local webserver --------------- From af082113c0c45af1519fbaa4466b5ee470095b2a Mon Sep 17 00:00:00 2001 From: aryankalantri Date: Sun, 22 Nov 2026 02:28:01 +0530 Subject: [PATCH 4/6] test(fs): cover gdrive protocol registration (#354) --- pydrive2/test/test_fs_registration.py | 31 +++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 pydrive2/test/test_fs_registration.py diff --git a/pydrive2/test/test_fs_registration.py b/pydrive2/test/test_fs_registration.py new file mode 100644 index 0000000..fdeb3e1 --- /dev/null +++ b/pydrive2/test/test_fs_registration.py @@ -0,0 +1,31 @@ +import importlib.metadata + +import fsspec +import pytest + +from pydrive2.fs import GDriveFileSystem + + +def test_protocol_is_gdrive(): + assert GDriveFileSystem.protocol == "gdrive" + + +def test_fsspec_entry_point_registered(): + entry_points = importlib.metadata.entry_points(group="fsspec") + gdrive = [ep for ep in entry_points if ep.name == "gdrive"] + assert len(gdrive) == 1 + assert gdrive[0].value == "pydrive2.fs:GDriveFileSystem" + + +def test_registry_resolves_gdrive(): + assert fsspec.registry.get_filesystem_class("gdrive") is GDriveFileSystem + + +def test_filesystem_without_credentials_reports_them(): + with pytest.raises(ValueError, match="Specify credentials"): + fsspec.filesystem("gdrive") + + +def test_path_defaults_to_drive_root(): + with pytest.raises(ValueError, match="Specify credentials"): + GDriveFileSystem() From 4615fa3f8d70c83992c741421c1ab20a3b0e4da3 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sat, 21 Nov 2026 16:07:35 -0800 Subject: [PATCH 5/6] Register the PyDrive2 fsspec backend and fix streamed file I/O --- docs/fsspec.rst | 42 +++++---- pydrive2/fs/spec.py | 111 +++++++++++++----------- pydrive2/fs/utils.py | 59 ++++++------- pydrive2/test/test_fs.py | 10 +++ pydrive2/test/test_fs_registration.py | 118 ++++++++++++++++++++++---- pydrive2/test/test_fs_unit.py | 27 ++++++ pyproject.toml | 4 +- 7 files changed, 253 insertions(+), 118 deletions(-) diff --git a/docs/fsspec.rst b/docs/fsspec.rst index f159270..cbbdd9b 100644 --- a/docs/fsspec.rst +++ b/docs/fsspec.rst @@ -11,29 +11,35 @@ Installation pip install 'pydrive2[fsspec]' -Once installed, ``fsspec`` finds the filesystem under the ``gdrive`` -protocol, so files can be opened without importing PyDrive2 first: +PyDrive2 registers the ``pydrive2://`` protocol with fsspec. The first +component is a case-sensitive Drive ID or one of the aliases ``root`` and +``appDataFolder``. Open an existing file by ID, or name a file relative to a +folder that the account can access: .. code-block:: python import fsspec - fs = fsspec.filesystem( - "gdrive", - "root", - client_id="my_client_id", -client_secret= - ) - - with fsspec.open( - "gdrive://root/path/to/file.txt", - client_id="my_client_id", -client_secret= - ) as f: - print(f.read()) - -Credentials are still required: without them the filesystem raises -``ValueError`` telling you which credential to supply. + options = { + "use_service_account": True, + "client_json_file_path": "/path/to/service-account.json", + } + with fsspec.open("pydrive2://FILE_ID", "rb", **options) as stream: + data = stream.read() + + with fsspec.open("pydrive2://FOLDER_ID/report.csv", "rt", **options) as stream: + header = stream.readline() + + fs = fsspec.filesystem("pydrive2", path="FOLDER_ID", **options) + names = fs.ls("FOLDER_ID") + +A filesystem instance is scoped to one root. Use URLs under the same folder +ID or root alias in a single ``fsspec.open_files`` call. Sibling URLs reuse +the filesystem and its directory ID cache. Reads by file ID skip name +lookups. Reads are sequential. To create a file, use a folder ID or root +alias followed by a filename. As with the direct PyDrive2 filesystem, +a write creates a Drive item even when a sibling has the same name. All +reads require credentials, including files shared by link. Local webserver --------------- diff --git a/pydrive2/fs/spec.py b/pydrive2/fs/spec.py index 51503e9..f1813eb 100644 --- a/pydrive2/fs/spec.py +++ b/pydrive2/fs/spec.py @@ -3,6 +3,7 @@ import logging import os import posixpath +import tempfile import threading from collections import defaultdict from contextlib import contextmanager @@ -191,7 +192,7 @@ def _service_auth( class GDriveFileSystem(AbstractFileSystem): """Access to gdrive as an fsspec filesystem""" - protocol = "gdrive" + protocol = "pydrive2" def __init__( self, @@ -274,12 +275,24 @@ def __init__( self._trash_only = trash_only self._acknowledge_abuse = acknowledge_abuse + @classmethod + def _get_kwargs_from_urls(cls, path): + # A filesystem caches names relative to a single Drive root. + root, _, _ = cls._strip_protocol(path).partition("/") + return {} if root in {"", "root"} else {"path": root} + def split_path(self, path): - parts = path.replace("//", "/").rstrip("/").split("/", 1) - if len(parts) == 2: - return parts - else: - return parts[0], "" + path = self._strip_protocol(path).replace("//", "/") + root, _, base = path.partition("/") + return root or "root", base + + def _split_mounted_path(self, path): + root, base = self.split_path(path) + if root != self.root: + raise ValueError( + f"Path {path!r} is outside Drive root {self.root!r}" + ) + return root, base @wrap_prop(threading.RLock()) @cached_property @@ -417,8 +430,7 @@ def _path_to_item_ids(self, path, create=False, use_cache=True): ) def _get_item_id(self, path, create=False, use_cache=True): - bucket, base = self.split_path(path) - assert bucket == self.root + _, base = self._split_mounted_path(path) item_ids = self._path_to_item_ids(base, create, use_cache) if item_ids: @@ -451,8 +463,18 @@ def _create_dir(self, parent_id, name, remote_path): return item["id"] def exists(self, path): + from pydrive2.files import ApiRequestError + + root, base = self._split_mounted_path(path) try: - self._get_item_id(path) + if not base and root not in {"root", "appDataFolder"}: + self.info(path) + else: + self._get_item_id(path) + except ApiRequestError as exc: + if exc.error.get("code") == 404: + return False + raise except FileNotFoundError: return False else: @@ -466,12 +488,11 @@ def info(self, path): gdrive_file.FetchMetadata(fields=_INFO_FIELDS) return _file_entry( - posixpath.join(bucket, base.rstrip("/")), gdrive_file + posixpath.join(bucket, base).rstrip("/"), gdrive_file ) def ls(self, path, detail=False): - bucket, base = self.split_path(path) - assert bucket == self.root + bucket, base = self._split_mounted_path(path) dir_ids = self._path_to_item_ids(base) if not dir_ids: @@ -491,8 +512,7 @@ def ls(self, path, detail=False): return [content["name"] for content in contents] def find(self, path, detail=False, **kwargs): - bucket, base = self.split_path(path) - assert bucket == self.root + bucket, base = self._split_mounted_path(path) # Make sure the base path is cached and dir_ids below has some # dirs revelant to this call @@ -641,21 +661,27 @@ def cb(value, total): gdrive_file.GetContentFile(rpath, **extra_args) - def _open(self, path, mode, **kwargs): - assert mode in {"rb", "wb"} + def _open(self, path, mode, block_size=None, **kwargs): + if mode not in {"rb", "wb"}: + raise NotImplementedError(f"Unsupported Drive file mode {mode!r}") if mode == "wb": + if not self._split_mounted_path(path)[1]: + raise ValueError( + "To create a Drive file, specify a folder ID and name" + ) return GDriveBufferedWriter(self, path) else: item_id = self._get_item_id(path) - return self._gdrive_open_file(item_id) + return self._gdrive_open_file(item_id, block_size) @_gdrive_retry - def _gdrive_open_file(self, item_id): + def _gdrive_open_file(self, item_id, block_size=None): param = {"id": item_id} # it does not create a file on the remote gdrive_file = self.client.CreateFile(param) + options = {"chunksize": block_size} if block_size else {} fd = gdrive_file.GetContentIOBuffer( - acknowledge_abuse=self._acknowledge_abuse + acknowledge_abuse=self._acknowledge_abuse, **options ) return IterStream(iter(fd)) @@ -699,43 +725,30 @@ def _gdrive_delete_file(self, item_id): raise -class GDriveBufferedWriter(io.IOBase): +class GDriveBufferedWriter(io.BufferedIOBase): + """Buffer a new Drive file locally until the outermost writer closes.""" + def __init__(self, fs, path): + super().__init__() self.fs = fs self.path = path - self.buffer = io.BytesIO() - self._closed = False - - def write(self, *args, **kwargs): - self.buffer.write(*args, **kwargs) + # The writer owns this stream until close(), including upload failures. + self.buffer = tempfile.SpooledTemporaryFile( # noqa: SIM115 + max_size=fs.blocksize + ) - def readable(self): - return False + def write(self, data): + return self.buffer.write(data) def writable(self): - return not self.readable() + return True - def flush(self): - self.buffer.flush() + def close(self): + if self.closed: + return try: + self.buffer.seek(0) self.fs.upload_fobj(self.buffer, self.path) finally: - self._closed = True - - def close(self): - if self._closed: - return - - self.flush() - self.buffer.close() - self._closed = True - - def __enter__(self): - return self - - def __exit__(self, *exc_info): - self.close() - - @property - def closed(self): - return self._closed + self.buffer.close() + super().close() diff --git a/pydrive2/fs/utils.py b/pydrive2/fs/utils.py index c2d3fe3..f7e5eb8 100644 --- a/pydrive2/fs/utils.py +++ b/pydrive2/fs/utils.py @@ -6,7 +6,7 @@ class IterStream(io.RawIOBase): def __init__(self, iterator): # pylint: disable=super-init-not-called self.iterator = iterator - self.leftover = b"" + self.leftover: bytes | memoryview = b"" def readable(self): return True @@ -14,43 +14,36 @@ def readable(self): def writable(self) -> bool: return False - # Python 3 requires only .readinto() method, it still uses other ones - # under some circumstances and falls back if those are absent. Since - # iterator already constructs byte strings for us, .readinto() is not the - # most optimal, so we provide .read1() too. + def _take(self, size): + if size == 0: + return b"" + chunk = self.leftover or next(self.iterator, b"") + if size < 0 or size >= len(chunk): + self.leftover = b"" + return chunk + # A Drive response can be much larger than the caller's buffer. + view = memoryview(chunk) + self.leftover = view[size:] + return view[:size] def readinto(self, b): - try: - n = len(b) # We're supposed to return at most this much - chunk = self.leftover or next(self.iterator) - output, self.leftover = chunk[:n], chunk[n:] - - n_out = len(output) - b[:n_out] = output - return n_out - except StopIteration: - return 0 # indicate EOF + chunk = self._take(len(b)) + b[: len(chunk)] = chunk + return len(chunk) readinto1 = readinto def read1(self, n=-1): - try: - chunk = self.leftover or next(self.iterator) - except StopIteration: - return b"" - - # Return an arbitrary number or bytes - if n <= 0: - self.leftover = b"" - return chunk - - output, self.leftover = chunk[:n], chunk[n:] - return output + return bytes(self._take(n)) def peek(self, n): - while len(self.leftover) < n: - try: - self.leftover += next(self.iterator) - except StopIteration: - break - return self.leftover[:n] + available = len(self.leftover) + if available < n: + chunks = [self.leftover] + for chunk in self.iterator: + chunks.append(chunk) + available += len(chunk) + if available >= n: + break + self.leftover = b"".join(chunks) + return bytes(self.leftover[:n]) diff --git a/pydrive2/test/test_fs.py b/pydrive2/test/test_fs.py index e013fa8..33029fa 100644 --- a/pydrive2/test/test_fs.py +++ b/pydrive2/test/test_fs.py @@ -327,6 +327,16 @@ def test_open_rw(fs, remote_dir): assert stream.read() == data +def test_fsspec_urls(fs, remote_dir): + options = {"google_auth": fs.client.auth} + named_url = f"pydrive2://{remote_dir}/report.txt" + with fsspec.open(named_url, "wt", **options) as stream: + stream.write("hello") + file_id = fs._get_item_id(remote_dir + "/report.txt") + with fsspec.open(f"pydrive2://{file_id}", "rt", **options) as stream: + assert stream.read() == "hello" + + def test_concurrent_operations(fs, fs_factory, remote_dir): # Include an extra dir name to force upload operations creating it # this way we can also test that only a single directory is created diff --git a/pydrive2/test/test_fs_registration.py b/pydrive2/test/test_fs_registration.py index fdeb3e1..cc53ca0 100644 --- a/pydrive2/test/test_fs_registration.py +++ b/pydrive2/test/test_fs_registration.py @@ -1,31 +1,117 @@ -import importlib.metadata +import subprocess +import sys +from urllib.parse import urlsplit import fsspec import pytest from pydrive2.fs import GDriveFileSystem +from pydrive2.fs.spec import FOLDER_MIME_TYPE +from pydrive2.test.http_mock import error_response, query, response -def test_protocol_is_gdrive(): - assert GDriveFileSystem.protocol == "gdrive" +def test_fsspec_discovers_backend_without_eager_import(): + subprocess.run( + [ + sys.executable, + "-c", + ( + "import fsspec, sys\n" + "assert 'pydrive2.fs' not in sys.modules\n" + "assert 'googleapiclient.discovery' not in sys.modules\n" + "cls = fsspec.get_filesystem_class('pydrive2')\n" + "from pydrive2.fs import GDriveFileSystem\n" + "assert cls is GDriveFileSystem" + ), + ], + check=True, + ) -def test_fsspec_entry_point_registered(): - entry_points = importlib.metadata.entry_points(group="fsspec") - gdrive = [ep for ep in entry_points if ep.name == "gdrive"] - assert len(gdrive) == 1 - assert gdrive[0].value == "pydrive2.fs:GDriveFileSystem" +def test_filesystem_factory(mock_drive): + with pytest.raises(ValueError, match="Specify credentials"): + fsspec.filesystem("pydrive2") + drive, requests = mock_drive() + fs = fsspec.filesystem("pydrive2", google_auth=drive.auth) + assert (fs.root, fs.base) == ("root", "") + mounted = GDriveFileSystem("FoLdEr/sub", google_auth=drive.auth) + assert (mounted.root, mounted.base) == ("FoLdEr", "sub") + assert not requests -def test_registry_resolves_gdrive(): - assert fsspec.registry.get_filesystem_class("gdrive") is GDriveFileSystem +def test_open_by_file_id(mock_drive): + drive, requests = mock_drive(({"status": "200"}, b"contents")) + with fsspec.open("pydrive2://FiLe_ID", google_auth=drive.auth) as stream: + assert stream.read() == b"contents" + (download,) = requests + assert urlsplit(download[0]).path == "/drive/v3/files/FiLe_ID" + assert query(download)["alt"] == ["media"] -def test_filesystem_without_credentials_reports_them(): - with pytest.raises(ValueError, match="Specify credentials"): - fsspec.filesystem("gdrive") +def test_open_files_reuses_folder_lookup(mock_drive): + def entry(name, item_id, mime_type="text/plain"): + return response( + {"files": [{"name": name, "id": item_id, "mimeType": mime_type}]} + ) + drive, requests = mock_drive( + response({"driveId": "shared"}), + entry("sub", "sub-ID", FOLDER_MIME_TYPE), + entry("a", "file-a"), + ({"status": "200"}, b"one"), + entry("b", "file-b"), + ({"status": "200"}, b"two"), + response({"files": []}), + ) + prefix = "pydrive2://FoLdEr/sub" + options = {"google_auth": drive.auth} + first, second = fsspec.open_files( + [prefix + "/a", prefix + "/b"], **options + ) + assert fsspec.open(prefix + "/b", **options).fs is first.fs is second.fs + assert not requests + with first as a, second as b: + assert (a.read(), b.read()) == (b"one", b"two") + assert first.fs.ls(prefix) == [] + shared, sub, a_lookup, _, b_lookup, _, listing = requests + assert urlsplit(shared[0]).path.endswith("/FoLdEr") + assert "'FoLdEr' in parents" in query(sub)["q"][0] + assert all( + "'sub-ID' in parents" in query(r)["q"][0] + for r in (a_lookup, b_lookup, listing) + ) + assert query(listing)["driveId"] == ["shared"] -def test_path_defaults_to_drive_root(): - with pytest.raises(ValueError, match="Specify credentials"): - GDriveFileSystem() + +def test_missing_file_id(mock_drive): + drive, requests = mock_drive(error_response(404, "notFound")) + fs, path = fsspec.core.url_to_fs( + "pydrive2://Missing", google_auth=drive.auth + ) + assert not fs.exists(path) + assert len(requests) == 1 + with pytest.raises(ValueError, match="folder ID and name"): + fs.open(path, "wb") + with pytest.raises(ValueError, match="outside Drive root"): + fs.ls("pydrive2://Other/sub") + + +@pytest.mark.parametrize("mode", ["wb", "wt"]) +def test_open_writes_once_on_close(mock_drive, monkeypatch, mode): + drive, _ = mock_drive() + uploads = [] + + def upload(fs, stream, path): + uploads.append((path, stream.read())) + + monkeypatch.setattr(GDriveFileSystem, "upload_fobj", upload) + monkeypatch.setattr(GDriveFileSystem, "blocksize", 3) + with fsspec.open( + "pydrive2://root/new", mode, google_auth=drive.auth + ) as writer: + value = b"data" if mode == "wb" else "data" + assert writer.write(value) == 4 + writer.flush() + assert not uploads + assert writer.write(value) == 4 + assert uploads == [("root/new", b"datadata")] diff --git a/pydrive2/test/test_fs_unit.py b/pydrive2/test/test_fs_unit.py index a2e1a25..8e06018 100644 --- a/pydrive2/test/test_fs_unit.py +++ b/pydrive2/test/test_fs_unit.py @@ -6,6 +6,7 @@ from pydrive2.fs import GDriveFileSystem from pydrive2.fs.spec import FOLDER_MIME_TYPE +from pydrive2.fs.utils import IterStream from pydrive2.test.http_mock import error_response, query, response @@ -203,3 +204,29 @@ def test_download_progress_uses_media_responses( assert all(query(request)["alt"] == ["media"] for request in downloads) assert updates == expected assert dest.read_bytes() == b"xyz" + + +def test_partial_reads_and_peek_across_chunks(): + stream = IterStream(iter([b"abc", b"de", b"fghi"])) + assert stream.read1(0) == b"" + assert stream.read1(1) == b"a" + assert stream.peek(5) == b"bcdef" + buf = bytearray(2) + assert stream.readinto(buf) == 2 + assert buf == b"bc" + assert stream.peek(2) == b"de" + assert stream.read1() == b"defghi" + assert stream.read() == b"" + + +def test_open_respects_download_block_size(mock_drive): + drive, requests = mock_drive( + ({"status": "206", "content-range": "bytes 0-1/3"}, b"xy"), + ({"status": "206", "content-range": "bytes 2-2/3"}, b"z"), + ) + with filesystem(drive, "FiLe").open("FiLe", block_size=2) as stream: + assert stream.read() == b"xyz" + assert [request[3]["range"] for request in requests] == [ + "bytes=0-1", + "bytes=2-3", + ] diff --git a/pyproject.toml b/pyproject.toml index 60dfa9b..95d137b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,8 +63,8 @@ typing = ["mypy==2.4.1", "types-httplib2", "types-PyYAML"] hook-dirs = "pydrive2.__pyinstaller:get_hook_dirs" tests = "pydrive2.__pyinstaller:get_PyInstaller_tests" -[project.entry-points.fsspec] -gdrive = "pydrive2.fs:GDriveFileSystem" +[project.entry-points."fsspec.specs"] +pydrive2 = "pydrive2.fs:GDriveFileSystem" [tool.setuptools] packages=[ From d407981e1279a7ab60990b86554678098dfe4307 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sat, 21 Nov 2026 16:22:09 -0800 Subject: [PATCH 6/6] Keep the fsspec registration PR focused on URL handling Restore the existing stream reader, download chunking, and buffered writer. Small-read copying and write buffering are separate concerns from registering a filesystem protocol. --- pydrive2/fs/spec.py | 60 ++++++++++++++++----------- pydrive2/fs/utils.py | 59 ++++++++++++++------------ pydrive2/test/test_fs_registration.py | 21 ---------- pydrive2/test/test_fs_unit.py | 27 ------------ 4 files changed, 68 insertions(+), 99 deletions(-) diff --git a/pydrive2/fs/spec.py b/pydrive2/fs/spec.py index f1813eb..3164010 100644 --- a/pydrive2/fs/spec.py +++ b/pydrive2/fs/spec.py @@ -3,7 +3,6 @@ import logging import os import posixpath -import tempfile import threading from collections import defaultdict from contextlib import contextmanager @@ -661,9 +660,8 @@ def cb(value, total): gdrive_file.GetContentFile(rpath, **extra_args) - def _open(self, path, mode, block_size=None, **kwargs): - if mode not in {"rb", "wb"}: - raise NotImplementedError(f"Unsupported Drive file mode {mode!r}") + def _open(self, path, mode, **kwargs): + assert mode in {"rb", "wb"} if mode == "wb": if not self._split_mounted_path(path)[1]: raise ValueError( @@ -672,16 +670,15 @@ def _open(self, path, mode, block_size=None, **kwargs): return GDriveBufferedWriter(self, path) else: item_id = self._get_item_id(path) - return self._gdrive_open_file(item_id, block_size) + return self._gdrive_open_file(item_id) @_gdrive_retry - def _gdrive_open_file(self, item_id, block_size=None): + def _gdrive_open_file(self, item_id): param = {"id": item_id} # it does not create a file on the remote gdrive_file = self.client.CreateFile(param) - options = {"chunksize": block_size} if block_size else {} fd = gdrive_file.GetContentIOBuffer( - acknowledge_abuse=self._acknowledge_abuse, **options + acknowledge_abuse=self._acknowledge_abuse ) return IterStream(iter(fd)) @@ -725,30 +722,43 @@ def _gdrive_delete_file(self, item_id): raise -class GDriveBufferedWriter(io.BufferedIOBase): - """Buffer a new Drive file locally until the outermost writer closes.""" - +class GDriveBufferedWriter(io.IOBase): def __init__(self, fs, path): - super().__init__() self.fs = fs self.path = path - # The writer owns this stream until close(), including upload failures. - self.buffer = tempfile.SpooledTemporaryFile( # noqa: SIM115 - max_size=fs.blocksize - ) + self.buffer = io.BytesIO() + self._closed = False + + def write(self, *args, **kwargs): + self.buffer.write(*args, **kwargs) - def write(self, data): - return self.buffer.write(data) + def readable(self): + return False def writable(self): - return True + return not self.readable() - def close(self): - if self.closed: - return + def flush(self): + self.buffer.flush() try: - self.buffer.seek(0) self.fs.upload_fobj(self.buffer, self.path) finally: - self.buffer.close() - super().close() + self._closed = True + + def close(self): + if self._closed: + return + + self.flush() + self.buffer.close() + self._closed = True + + def __enter__(self): + return self + + def __exit__(self, *exc_info): + self.close() + + @property + def closed(self): + return self._closed diff --git a/pydrive2/fs/utils.py b/pydrive2/fs/utils.py index f7e5eb8..c2d3fe3 100644 --- a/pydrive2/fs/utils.py +++ b/pydrive2/fs/utils.py @@ -6,7 +6,7 @@ class IterStream(io.RawIOBase): def __init__(self, iterator): # pylint: disable=super-init-not-called self.iterator = iterator - self.leftover: bytes | memoryview = b"" + self.leftover = b"" def readable(self): return True @@ -14,36 +14,43 @@ def readable(self): def writable(self) -> bool: return False - def _take(self, size): - if size == 0: - return b"" - chunk = self.leftover or next(self.iterator, b"") - if size < 0 or size >= len(chunk): - self.leftover = b"" - return chunk - # A Drive response can be much larger than the caller's buffer. - view = memoryview(chunk) - self.leftover = view[size:] - return view[:size] + # Python 3 requires only .readinto() method, it still uses other ones + # under some circumstances and falls back if those are absent. Since + # iterator already constructs byte strings for us, .readinto() is not the + # most optimal, so we provide .read1() too. def readinto(self, b): - chunk = self._take(len(b)) - b[: len(chunk)] = chunk - return len(chunk) + try: + n = len(b) # We're supposed to return at most this much + chunk = self.leftover or next(self.iterator) + output, self.leftover = chunk[:n], chunk[n:] + + n_out = len(output) + b[:n_out] = output + return n_out + except StopIteration: + return 0 # indicate EOF readinto1 = readinto def read1(self, n=-1): - return bytes(self._take(n)) + try: + chunk = self.leftover or next(self.iterator) + except StopIteration: + return b"" + + # Return an arbitrary number or bytes + if n <= 0: + self.leftover = b"" + return chunk + + output, self.leftover = chunk[:n], chunk[n:] + return output def peek(self, n): - available = len(self.leftover) - if available < n: - chunks = [self.leftover] - for chunk in self.iterator: - chunks.append(chunk) - available += len(chunk) - if available >= n: - break - self.leftover = b"".join(chunks) - return bytes(self.leftover[:n]) + while len(self.leftover) < n: + try: + self.leftover += next(self.iterator) + except StopIteration: + break + return self.leftover[:n] diff --git a/pydrive2/test/test_fs_registration.py b/pydrive2/test/test_fs_registration.py index cc53ca0..58a8961 100644 --- a/pydrive2/test/test_fs_registration.py +++ b/pydrive2/test/test_fs_registration.py @@ -94,24 +94,3 @@ def test_missing_file_id(mock_drive): fs.open(path, "wb") with pytest.raises(ValueError, match="outside Drive root"): fs.ls("pydrive2://Other/sub") - - -@pytest.mark.parametrize("mode", ["wb", "wt"]) -def test_open_writes_once_on_close(mock_drive, monkeypatch, mode): - drive, _ = mock_drive() - uploads = [] - - def upload(fs, stream, path): - uploads.append((path, stream.read())) - - monkeypatch.setattr(GDriveFileSystem, "upload_fobj", upload) - monkeypatch.setattr(GDriveFileSystem, "blocksize", 3) - with fsspec.open( - "pydrive2://root/new", mode, google_auth=drive.auth - ) as writer: - value = b"data" if mode == "wb" else "data" - assert writer.write(value) == 4 - writer.flush() - assert not uploads - assert writer.write(value) == 4 - assert uploads == [("root/new", b"datadata")] diff --git a/pydrive2/test/test_fs_unit.py b/pydrive2/test/test_fs_unit.py index 8e06018..a2e1a25 100644 --- a/pydrive2/test/test_fs_unit.py +++ b/pydrive2/test/test_fs_unit.py @@ -6,7 +6,6 @@ from pydrive2.fs import GDriveFileSystem from pydrive2.fs.spec import FOLDER_MIME_TYPE -from pydrive2.fs.utils import IterStream from pydrive2.test.http_mock import error_response, query, response @@ -204,29 +203,3 @@ def test_download_progress_uses_media_responses( assert all(query(request)["alt"] == ["media"] for request in downloads) assert updates == expected assert dest.read_bytes() == b"xyz" - - -def test_partial_reads_and_peek_across_chunks(): - stream = IterStream(iter([b"abc", b"de", b"fghi"])) - assert stream.read1(0) == b"" - assert stream.read1(1) == b"a" - assert stream.peek(5) == b"bcdef" - buf = bytearray(2) - assert stream.readinto(buf) == 2 - assert buf == b"bc" - assert stream.peek(2) == b"de" - assert stream.read1() == b"defghi" - assert stream.read() == b"" - - -def test_open_respects_download_block_size(mock_drive): - drive, requests = mock_drive( - ({"status": "206", "content-range": "bytes 0-1/3"}, b"xy"), - ({"status": "206", "content-range": "bytes 2-2/3"}, b"z"), - ) - with filesystem(drive, "FiLe").open("FiLe", block_size=2) as stream: - assert stream.read() == b"xyz" - assert [request[3]["range"] for request in requests] == [ - "bytes=0-1", - "bytes=2-3", - ]