Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/fsspec.rst
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,12 @@ Using filesystem
for fname in fnames:
print(f"file: {root}/{fname}")

For sequential reads, pass a positive ``block_size`` to ``fs.open()`` to set
the Drive download request size in bytes. A writer buffers data locally,
using a temporary file for larger uploads. Calling ``flush()`` flushes the
local buffer. Closing the writer uploads the file to Drive and reports any
upload error.

Filesystem instance offers a large number of methods for getting information
about and manipulating files, refer to fsspec docs on
`how to use a filesystem`_.
Expand Down
68 changes: 35 additions & 33 deletions pydrive2/fs/spec.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import logging
import os
import posixpath
import tempfile
import threading
from collections import defaultdict
from contextlib import contextmanager
Expand Down Expand Up @@ -638,21 +639,24 @@ def cb(value, total):

gdrive_file.GetContentFile(rpath, **extra_args)

def _open(self, path, mode, **kwargs):
def _open(self, path, mode, block_size=None, **kwargs):
assert mode in {"rb", "wb"}
if mode == "wb":
return GDriveBufferedWriter(self, path)
else:
item_id = self._get_item_id(path)
return self._gdrive_open_file(item_id)
return self._gdrive_open_file(item_id, block_size)

@_gdrive_retry
def _gdrive_open_file(self, item_id):
def _gdrive_open_file(self, item_id, block_size=None):
param = {"id": item_id}
# it does not create a file on the remote
gdrive_file = self.client.CreateFile(param)
options = {}
if block_size not in (None, "default") and block_size > 0:
options["chunksize"] = block_size
fd = gdrive_file.GetContentIOBuffer(
acknowledge_abuse=self._acknowledge_abuse
acknowledge_abuse=self._acknowledge_abuse, **options
)
return IterStream(iter(fd))

Expand Down Expand Up @@ -696,43 +700,41 @@ def _gdrive_delete_file(self, item_id):
raise


class GDriveBufferedWriter(io.IOBase):
class GDriveBufferedWriter(io.BufferedIOBase):
"""Stage a seekable body for Drive's resumable upload and send it on close."""

def __init__(self, fs, path):
super().__init__()
self.fs = fs
self.path = path
self.buffer = io.BytesIO()
self._closed = False

def write(self, *args, **kwargs):
self.buffer.write(*args, **kwargs)

def readable(self):
return False
self._memory_limit: int | None = fs.blocksize
# The writer owns the spool until close(), including failed uploads.
self.buffer = tempfile.SpooledTemporaryFile( # noqa: SIM115
max_size=fs.blocksize
)

def writable(self):
return not self.readable()
return self.buffer.writable()

def write(self, data):
self._checkClosed()
# Roll before copying data into BytesIO. Once on disk, skip the size check.
if self._memory_limit is not None and (
self.buffer.tell() + memoryview(data).nbytes > self._memory_limit
):
self.buffer.rollover()
self._memory_limit = None
return self.buffer.write(data)
Comment thread
shcheklein marked this conversation as resolved.
Resolved

def flush(self):
self.buffer.flush()
try:
self.fs.upload_fobj(self.buffer, self.path)
finally:
self._closed = True

def close(self):
if self._closed:
if self.closed:
return

self.flush()
self.buffer.close()
self._closed = True

def __enter__(self):
return self

def __exit__(self, *exc_info):
self.close()

@property
def closed(self):
return self._closed
with self.buffer:
try:
self.buffer.seek(0)
self.fs.upload_fobj(self.buffer, self.path)
finally:
super().close()
83 changes: 46 additions & 37 deletions pydrive2/fs/utils.py
Original file line number Diff line number Diff line change
@@ -1,56 +1,65 @@
import io
from typing import TYPE_CHECKING

if TYPE_CHECKING:
from collections.abc import Iterator


class IterStream(io.RawIOBase):
"""Wraps an iterator yielding bytes as a file object"""
"""Wraps an iterator yielding bytes as a file object."""

def __init__(self, iterator): # pylint: disable=super-init-not-called
self.iterator = iterator
self.leftover = b""
def __init__(self, iterator):
super().__init__()
self.iterator: Iterator[bytes] = filter(None, iterator)
self.leftover: bytes | memoryview = b""

def readable(self):
return True

def writable(self) -> bool:
return False
def close(self):
self.leftover = b""
self.iterator = iter(())
super().close()

# Python 3 requires only .readinto() method, it still uses other ones
# under some circumstances and falls back if those are absent. Since
# iterator already constructs byte strings for us, .readinto() is not the
# most optimal, so we provide .read1() too.
def _take(self, size):
if size == 0:
return b""
chunk = self.leftover or next(self.iterator, b"")
if size < 0 or size >= len(chunk):
self.leftover = b""
return chunk
# Keep a view of the unread bytes until the response is exhausted.
view = memoryview(chunk)
self.leftover = view[size:]
return view[:size]

def readinto(self, b):
try:
n = len(b) # We're supposed to return at most this much
chunk = self.leftover or next(self.iterator)
output, self.leftover = chunk[:n], chunk[n:]

n_out = len(output)
b[:n_out] = output
return n_out
except StopIteration:
return 0 # indicate EOF
self._checkClosed()
with memoryview(b).cast("B") as dest:
if dest.readonly:
raise TypeError("readinto() requires a writable buffer")
chunk = self._take(len(dest))
dest[: len(chunk)] = chunk
return len(chunk)

readinto1 = readinto

def read1(self, n=-1):
try:
chunk = self.leftover or next(self.iterator)
except StopIteration:
return b""

# Return an arbitrary number or bytes
if n <= 0:
self.leftover = b""
return chunk

output, self.leftover = chunk[:n], chunk[n:]
return output
self._checkClosed()
return bytes(self._take(n))

def peek(self, n):
while len(self.leftover) < n:
self._checkClosed()
available = len(self.leftover)
if available < n:
chunks = [self.leftover] if available else []
try:
self.leftover += next(self.iterator)
except StopIteration:
break
return self.leftover[:n]
for chunk in self.iterator:
chunks.append(chunk)
available += len(chunk)
if available >= n:
break
finally:
# Preserve fetched bytes even if a later request fails.
self.leftover = b"".join(chunks)
return bytes(self.leftover[:n])
4 changes: 3 additions & 1 deletion pydrive2/test/test_fs.py
Original file line number Diff line number Diff line change
Expand Up @@ -321,7 +321,9 @@ def test_open_rw(fs, remote_dir):
data = b"dvc.org"

with fs.open(remote_dir + "/a.txt", "wb") as stream:
stream.write(data)
stream.write(data[:3])
stream.flush()
stream.write(data[3:])

with fs.open(remote_dir + "/a.txt") as stream:
assert stream.read() == data
Expand Down
141 changes: 141 additions & 0 deletions pydrive2/test/test_fs_unit.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
import json
import tempfile
from urllib.parse import urlsplit

import pytest
from fsspec.callbacks import Callback
from googleapiclient.http import DEFAULT_CHUNK_SIZE

from pydrive2.fs import GDriveFileSystem
from pydrive2.fs.spec import FOLDER_MIME_TYPE
from pydrive2.fs.utils import IterStream
from pydrive2.test.http_mock import error_response, query, response


Expand Down Expand Up @@ -203,3 +206,141 @@ def test_download_progress_uses_media_responses(
assert all(query(request)["alt"] == ["media"] for request in downloads)
assert updates == expected
assert dest.read_bytes() == b"xyz"


def test_partial_reads_and_peek_across_chunks():
stream = IterStream(iter([b"abc", b"", b"de", b"fghi"]))
assert stream.read1(0) == b""
assert stream.readinto(bytearray()) == 0
assert stream.read1(1) == b"a"
assert stream.peek(5) == b"bcdef"
buf = bytearray(2)
assert stream.readinto1(buf) == 2
assert buf == b"bc"
assert stream.peek(2) == b"de"
assert stream.read() == b"defghi"
assert stream.read1() == b""
assert stream.peek(4) == b""


def test_peek_preserves_buffer_if_next_chunk_fails():
def chunks():
yield b"abc"
raise OSError("download failed")

with IterStream(chunks()) as stream:
with pytest.raises(OSError, match="download failed"):
stream.peek(4)
assert stream.read1() == b"abc"
with pytest.raises(ValueError, match="closed"):
stream.read1()
with pytest.raises(ValueError, match="closed"):
stream.readinto(b"")


def test_readinto_releases_destination_on_download_error():
def chunks():
yield b"x"
raise OSError("download failed")

dest = bytearray(1)
with IterStream(chunks()) as stream:
stream.readinto(dest)
with pytest.raises(OSError) as error:
stream.readinto(dest)
# Keep the download traceback alive while the caller resizes its buffer.
dest.clear()
assert str(error.value) == "download failed"


def test_open_uses_requested_download_block_size(mock_drive):
drive, requests = mock_drive(
({"status": "206", "content-range": "bytes 0-1/3"}, b"xy"),
({"status": "206", "content-range": "bytes 2-2/3"}, b"z"),
)
with filesystem(drive, "FileID").open("FileID", block_size=2) as stream:
assert stream.read() == b"xyz"
assert [r[3]["range"] for r in requests] == ["bytes=0-1", "bytes=2-3"]


@pytest.mark.parametrize("block_size", [None, "default"])
def test_open_uses_default_download_block_size(mock_drive, block_size):
drive, requests = mock_drive(({"status": "200"}, b"data"))
fs = filesystem(drive, "FileID")
with fs.open("FileID", block_size=block_size) as stream:
assert stream.read() == b"data"
(request,) = requests
assert request[3]["range"] == f"bytes=0-{DEFAULT_CHUNK_SIZE - 1}"


@pytest.mark.parametrize("mode", ["wb", "wt"])
def test_flush_then_write_uploads_once_on_close(mock_drive, monkeypatch, mode):
drive, _ = mock_drive()
fs = filesystem(drive)
monkeypatch.setattr(fs, "blocksize", 5)
rollover_positions = []
rollover = tempfile.SpooledTemporaryFile.rollover

def record_rollover(spool):
rollover_positions.append(spool.tell())
return rollover(spool)

monkeypatch.setattr(
tempfile.SpooledTemporaryFile, "rollover", record_rollover
)
uploads = []

def upload(body, path):
uploads.append((path, body.read()))

monkeypatch.setattr(fs, "upload_fobj", upload)
with fs.open("root/file", mode) as writer:
value = b"abc" if mode == "wb" else "abc"
written = writer.write(value)
assert written == 3
writer.flush()
assert not writer.closed
assert not uploads
assert not rollover_positions
written = writer.write(value)
assert written == 3
assert rollover_positions == [3]
writer.close()
assert uploads == [("root/file", b"abcabc")]


def test_failed_close_releases_spool_and_propagates_error(
mock_drive, monkeypatch
):
drive, _ = mock_drive()
fs = filesystem(drive)
monkeypatch.setattr(fs, "blocksize", 2)
attempts = []

def fail(body, path):
attempts.append(body)
assert body.read() == b"data"
raise OSError("upload failed")

monkeypatch.setattr(fs, "upload_fobj", fail)
writer = fs.open("root/file", "wb")
writer.write(b"data")
with pytest.raises(OSError, match="upload failed"):
writer.close()
assert writer.closed
writer.close()
(body,) = attempts
assert body.closed


def test_large_write_after_close_does_not_open_a_tempfile(mock_drive, mocker):
drive, _ = mock_drive()
fs = filesystem(drive)
mocker.patch.object(fs, "blocksize", 2)
mocker.patch.object(fs, "upload_fobj")
backing_file = mocker.spy(tempfile, "TemporaryFile")
with fs.open("root/file", "wb") as writer:
writer.write(b"x")
with pytest.raises(ValueError, match="closed"):
writer.write(b"too large")
backing_file.assert_not_called()
Loading