diff --git a/docs/fsspec.rst b/docs/fsspec.rst index 516abeb..bce831b 100644 --- a/docs/fsspec.rst +++ b/docs/fsspec.rst @@ -129,6 +129,12 @@ Using filesystem for fname in fnames: print(f"file: {root}/{fname}") +For sequential reads, pass a positive ``block_size`` to ``fs.open()`` to set +the Drive download request size in bytes. A writer buffers data locally, +using a temporary file for larger uploads. Calling ``flush()`` flushes the +local buffer. Closing the writer uploads the file to Drive and reports any +upload error. + Filesystem instance offers a large number of methods for getting information about and manipulating files, refer to fsspec docs on `how to use a filesystem`_. diff --git a/pydrive2/fs/spec.py b/pydrive2/fs/spec.py index 6166662..8c9e9f0 100644 --- a/pydrive2/fs/spec.py +++ b/pydrive2/fs/spec.py @@ -3,6 +3,7 @@ import logging import os import posixpath +import tempfile import threading from collections import defaultdict from contextlib import contextmanager @@ -638,21 +639,24 @@ def cb(value, total): gdrive_file.GetContentFile(rpath, **extra_args) - def _open(self, path, mode, **kwargs): + def _open(self, path, mode, block_size=None, **kwargs): assert mode in {"rb", "wb"} if mode == "wb": return GDriveBufferedWriter(self, path) else: item_id = self._get_item_id(path) - return self._gdrive_open_file(item_id) + return self._gdrive_open_file(item_id, block_size) @_gdrive_retry - def _gdrive_open_file(self, item_id): + def _gdrive_open_file(self, item_id, block_size=None): param = {"id": item_id} # it does not create a file on the remote gdrive_file = self.client.CreateFile(param) + options = {} + if block_size not in (None, "default") and block_size > 0: + options["chunksize"] = block_size fd = gdrive_file.GetContentIOBuffer( - acknowledge_abuse=self._acknowledge_abuse + acknowledge_abuse=self._acknowledge_abuse, **options ) return IterStream(iter(fd)) @@ -696,43 +700,41 @@ def _gdrive_delete_file(self, item_id): raise -class GDriveBufferedWriter(io.IOBase): +class GDriveBufferedWriter(io.BufferedIOBase): + """Stage a seekable body for Drive's resumable upload and send it on close.""" + def __init__(self, fs, path): + super().__init__() self.fs = fs self.path = path - self.buffer = io.BytesIO() - self._closed = False - - def write(self, *args, **kwargs): - self.buffer.write(*args, **kwargs) - - def readable(self): - return False + self._memory_limit: int | None = fs.blocksize + # The writer owns the spool until close(), including failed uploads. + self.buffer = tempfile.SpooledTemporaryFile( # noqa: SIM115 + max_size=fs.blocksize + ) def writable(self): - return not self.readable() + return self.buffer.writable() + + def write(self, data): + self._checkClosed() + # Roll before copying data into BytesIO. Once on disk, skip the size check. + if self._memory_limit is not None and ( + self.buffer.tell() + memoryview(data).nbytes > self._memory_limit + ): + self.buffer.rollover() + self._memory_limit = None + return self.buffer.write(data) def flush(self): self.buffer.flush() - try: - self.fs.upload_fobj(self.buffer, self.path) - finally: - self._closed = True def close(self): - if self._closed: + if self.closed: return - - self.flush() - self.buffer.close() - self._closed = True - - def __enter__(self): - return self - - def __exit__(self, *exc_info): - self.close() - - @property - def closed(self): - return self._closed + with self.buffer: + try: + self.buffer.seek(0) + self.fs.upload_fobj(self.buffer, self.path) + finally: + super().close() diff --git a/pydrive2/fs/utils.py b/pydrive2/fs/utils.py index c2d3fe3..5af2d0b 100644 --- a/pydrive2/fs/utils.py +++ b/pydrive2/fs/utils.py @@ -1,56 +1,65 @@ import io +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterator class IterStream(io.RawIOBase): - """Wraps an iterator yielding bytes as a file object""" + """Wraps an iterator yielding bytes as a file object.""" - def __init__(self, iterator): # pylint: disable=super-init-not-called - self.iterator = iterator - self.leftover = b"" + def __init__(self, iterator): + super().__init__() + self.iterator: Iterator[bytes] = filter(None, iterator) + self.leftover: bytes | memoryview = b"" def readable(self): return True - def writable(self) -> bool: - return False + def close(self): + self.leftover = b"" + self.iterator = iter(()) + super().close() - # Python 3 requires only .readinto() method, it still uses other ones - # under some circumstances and falls back if those are absent. Since - # iterator already constructs byte strings for us, .readinto() is not the - # most optimal, so we provide .read1() too. + def _take(self, size): + if size == 0: + return b"" + chunk = self.leftover or next(self.iterator, b"") + if size < 0 or size >= len(chunk): + self.leftover = b"" + return chunk + # Keep a view of the unread bytes until the response is exhausted. + view = memoryview(chunk) + self.leftover = view[size:] + return view[:size] def readinto(self, b): - try: - n = len(b) # We're supposed to return at most this much - chunk = self.leftover or next(self.iterator) - output, self.leftover = chunk[:n], chunk[n:] - - n_out = len(output) - b[:n_out] = output - return n_out - except StopIteration: - return 0 # indicate EOF + self._checkClosed() + with memoryview(b).cast("B") as dest: + if dest.readonly: + raise TypeError("readinto() requires a writable buffer") + chunk = self._take(len(dest)) + dest[: len(chunk)] = chunk + return len(chunk) readinto1 = readinto def read1(self, n=-1): - try: - chunk = self.leftover or next(self.iterator) - except StopIteration: - return b"" - - # Return an arbitrary number or bytes - if n <= 0: - self.leftover = b"" - return chunk - - output, self.leftover = chunk[:n], chunk[n:] - return output + self._checkClosed() + return bytes(self._take(n)) def peek(self, n): - while len(self.leftover) < n: + self._checkClosed() + available = len(self.leftover) + if available < n: + chunks = [self.leftover] if available else [] try: - self.leftover += next(self.iterator) - except StopIteration: - break - return self.leftover[:n] + for chunk in self.iterator: + chunks.append(chunk) + available += len(chunk) + if available >= n: + break + finally: + # Preserve fetched bytes even if a later request fails. + self.leftover = b"".join(chunks) + return bytes(self.leftover[:n]) diff --git a/pydrive2/test/test_fs.py b/pydrive2/test/test_fs.py index e013fa8..dc2409d 100644 --- a/pydrive2/test/test_fs.py +++ b/pydrive2/test/test_fs.py @@ -321,7 +321,9 @@ def test_open_rw(fs, remote_dir): data = b"dvc.org" with fs.open(remote_dir + "/a.txt", "wb") as stream: - stream.write(data) + stream.write(data[:3]) + stream.flush() + stream.write(data[3:]) with fs.open(remote_dir + "/a.txt") as stream: assert stream.read() == data diff --git a/pydrive2/test/test_fs_unit.py b/pydrive2/test/test_fs_unit.py index a2e1a25..4251361 100644 --- a/pydrive2/test/test_fs_unit.py +++ b/pydrive2/test/test_fs_unit.py @@ -1,11 +1,14 @@ import json +import tempfile from urllib.parse import urlsplit import pytest from fsspec.callbacks import Callback +from googleapiclient.http import DEFAULT_CHUNK_SIZE from pydrive2.fs import GDriveFileSystem from pydrive2.fs.spec import FOLDER_MIME_TYPE +from pydrive2.fs.utils import IterStream from pydrive2.test.http_mock import error_response, query, response @@ -203,3 +206,141 @@ def test_download_progress_uses_media_responses( assert all(query(request)["alt"] == ["media"] for request in downloads) assert updates == expected assert dest.read_bytes() == b"xyz" + + +def test_partial_reads_and_peek_across_chunks(): + stream = IterStream(iter([b"abc", b"", b"de", b"fghi"])) + assert stream.read1(0) == b"" + assert stream.readinto(bytearray()) == 0 + assert stream.read1(1) == b"a" + assert stream.peek(5) == b"bcdef" + buf = bytearray(2) + assert stream.readinto1(buf) == 2 + assert buf == b"bc" + assert stream.peek(2) == b"de" + assert stream.read() == b"defghi" + assert stream.read1() == b"" + assert stream.peek(4) == b"" + + +def test_peek_preserves_buffer_if_next_chunk_fails(): + def chunks(): + yield b"abc" + raise OSError("download failed") + + with IterStream(chunks()) as stream: + with pytest.raises(OSError, match="download failed"): + stream.peek(4) + assert stream.read1() == b"abc" + with pytest.raises(ValueError, match="closed"): + stream.read1() + with pytest.raises(ValueError, match="closed"): + stream.readinto(b"") + + +def test_readinto_releases_destination_on_download_error(): + def chunks(): + yield b"x" + raise OSError("download failed") + + dest = bytearray(1) + with IterStream(chunks()) as stream: + stream.readinto(dest) + with pytest.raises(OSError) as error: + stream.readinto(dest) + # Keep the download traceback alive while the caller resizes its buffer. + dest.clear() + assert str(error.value) == "download failed" + + +def test_open_uses_requested_download_block_size(mock_drive): + drive, requests = mock_drive( + ({"status": "206", "content-range": "bytes 0-1/3"}, b"xy"), + ({"status": "206", "content-range": "bytes 2-2/3"}, b"z"), + ) + with filesystem(drive, "FileID").open("FileID", block_size=2) as stream: + assert stream.read() == b"xyz" + assert [r[3]["range"] for r in requests] == ["bytes=0-1", "bytes=2-3"] + + +@pytest.mark.parametrize("block_size", [None, "default"]) +def test_open_uses_default_download_block_size(mock_drive, block_size): + drive, requests = mock_drive(({"status": "200"}, b"data")) + fs = filesystem(drive, "FileID") + with fs.open("FileID", block_size=block_size) as stream: + assert stream.read() == b"data" + (request,) = requests + assert request[3]["range"] == f"bytes=0-{DEFAULT_CHUNK_SIZE - 1}" + + +@pytest.mark.parametrize("mode", ["wb", "wt"]) +def test_flush_then_write_uploads_once_on_close(mock_drive, monkeypatch, mode): + drive, _ = mock_drive() + fs = filesystem(drive) + monkeypatch.setattr(fs, "blocksize", 5) + rollover_positions = [] + rollover = tempfile.SpooledTemporaryFile.rollover + + def record_rollover(spool): + rollover_positions.append(spool.tell()) + return rollover(spool) + + monkeypatch.setattr( + tempfile.SpooledTemporaryFile, "rollover", record_rollover + ) + uploads = [] + + def upload(body, path): + uploads.append((path, body.read())) + + monkeypatch.setattr(fs, "upload_fobj", upload) + with fs.open("root/file", mode) as writer: + value = b"abc" if mode == "wb" else "abc" + written = writer.write(value) + assert written == 3 + writer.flush() + assert not writer.closed + assert not uploads + assert not rollover_positions + written = writer.write(value) + assert written == 3 + assert rollover_positions == [3] + writer.close() + assert uploads == [("root/file", b"abcabc")] + + +def test_failed_close_releases_spool_and_propagates_error( + mock_drive, monkeypatch +): + drive, _ = mock_drive() + fs = filesystem(drive) + monkeypatch.setattr(fs, "blocksize", 2) + attempts = [] + + def fail(body, path): + attempts.append(body) + assert body.read() == b"data" + raise OSError("upload failed") + + monkeypatch.setattr(fs, "upload_fobj", fail) + writer = fs.open("root/file", "wb") + writer.write(b"data") + with pytest.raises(OSError, match="upload failed"): + writer.close() + assert writer.closed + writer.close() + (body,) = attempts + assert body.closed + + +def test_large_write_after_close_does_not_open_a_tempfile(mock_drive, mocker): + drive, _ = mock_drive() + fs = filesystem(drive) + mocker.patch.object(fs, "blocksize", 2) + mocker.patch.object(fs, "upload_fobj") + backing_file = mocker.spy(tempfile, "TemporaryFile") + with fs.open("root/file", "wb") as writer: + writer.write(b"x") + with pytest.raises(ValueError, match="closed"): + writer.write(b"too large") + backing_file.assert_not_called()