From c599c404db190a49f6fc335f3abd35bf58a7c4c6 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sat, 21 Nov 2026 11:46:04 -0800 Subject: [PATCH 1/2] Copy files on Drive and reduce metadata and export requests --- pydrive2/files.py | 140 ++++++++++++++------------------- pydrive2/fs/spec.py | 33 +++++--- pydrive2/test/test_drive_v3.py | 100 +++++++++++++++++++++++ pydrive2/test/test_fs.py | 9 +++ 4 files changed, 188 insertions(+), 94 deletions(-) diff --git a/pydrive2/files.py b/pydrive2/files.py index 68b2151..095eb31 100644 --- a/pydrive2/files.py +++ b/pydrive2/files.py @@ -24,6 +24,15 @@ "resourceKey,webViewLink,createdTime,modifiedTime" ) +WORKSPACE_EXPORT_TYPES = frozenset( + { + "application/vnd.google-apps.document", + "application/vnd.google-apps.spreadsheet", + "application/vnd.google-apps.presentation", + "application/vnd.google-apps.drawing", + } +) + BLOCK_SIZE = 1024 # Usage: MIME_TYPE_TO_BOM['']['']. MIME_TYPE_TO_BOM = { @@ -339,57 +348,25 @@ def GetContentFile( :type acknowledge_abuse: bool :raises: ApiRequestError, FileNotUploadedError """ - files = self.auth.service.files() - file_id = self.metadata.get("id") or self.get("id") - if not file_id: - raise FileNotUploadedError() - def download(fd, request): - downloader = MediaIoBaseDownload( - fd, self._WrapRequest(request), chunksize=chunksize - ) - done = False - while done is False: - status, done = downloader.next_chunk() - if callback: - callback(status.resumable_progress, status.total_size) - - with open(filename, mode="w+b") as fd: - # Should use files.export_media instead of files.get_media if - # metadata["mimeType"].startswith("application/vnd.google-apps."). - # But that would first require a slow call to FetchMetadata(). - # We prefer to try-except for speed. - try: - download( - fd, - files.get_media( - fileId=file_id, - acknowledgeAbuse=acknowledge_abuse, - supportsAllDrives=True, - ), + def download(request, export_type): + with open(filename, mode="w+b") as fd: + downloader = MediaIoBaseDownload( + fd, request, chunksize=chunksize ) - except errors.HttpError as error: - exc = ApiRequestError(error) - if ( - exc.error["code"] != 403 - or exc.GetField("reason") != "fileNotDownloadable" - ): - raise exc - mimetype = mimetype or "text/plain" - fd.seek(0) # just in case `download()` modified `fd` - try: - download( - fd, - files.export_media(fileId=file_id, mimeType=mimetype), - ) - except errors.HttpError as error: - raise ApiRequestError(error) - - if mimetype == "text/plain" and remove_bom: - fd.seek(0) - bom = self._GetBOM(mimetype) - if bom: - self._RemovePrefix(fd, bom) + done = False + while not done: + try: + status, done = downloader.next_chunk() + except errors.HttpError as error: + raise ApiRequestError(error) from error + if callback: + callback(status.resumable_progress, status.total_size) + if remove_bom and (export_type or mimetype) == "text/plain": + fd.seek(0) + self._RemovePrefix(fd, self._GetBOM("text/plain")) + + self._DownloadContent(download, mimetype, acknowledge_abuse) @LoadAuth def GetContentIOBuffer( @@ -416,48 +393,47 @@ def GetContentIOBuffer( :returns: MediaIoReadable -- file-like object. :raises: ApiRequestError, FileNotUploadedError """ - files = self.auth.service.files() - file_id = self.metadata.get("id") or self.get("id") - if not file_id: - raise FileNotUploadedError() - # Should use files.export_media instead of files.get_media if - # metadata["mimeType"].startswith("application/vnd.google-apps."). - # But that would first require a slow call to FetchMetadata(). - # We prefer to try-except for speed. - try: - request = self._WrapRequest( - files.get_media( - fileId=file_id, - acknowledgeAbuse=acknowledge_abuse, - supportsAllDrives=True, - ) - ) - return MediaIoReadable( - request, encoding=encoding, chunksize=chunksize - ) - except ApiRequestError as exc: - if ( - exc.error["code"] != 403 - or exc.GetField("reason") != "fileNotDownloadable" - ): - raise - mimetype = mimetype or "text/plain" - request = self._WrapRequest( - files.export_media(fileId=file_id, mimeType=mimetype) - ) - remove_prefix = ( - self._GetBOM(mimetype) - if mimetype == "text/plain" and remove_bom + def open_stream(request, export_type): + prefix = ( + self._GetBOM("text/plain") + if remove_bom and export_type == "text/plain" else b"" ) return MediaIoReadable( request, encoding=encoding, - remove_prefix=remove_prefix, + remove_prefix=prefix, chunksize=chunksize, ) + return self._DownloadContent(open_stream, mimetype, acknowledge_abuse) + + def _DownloadContent(self, consume, mimetype, acknowledge_abuse): + file_id = self.metadata.get("id") or self.get("id") + if not file_id: + raise FileNotUploadedError() + + files = self.auth.service.files() + if self.metadata.get("mimeType") not in WORKSPACE_EXPORT_TYPES: + request = files.get_media( + fileId=file_id, + acknowledgeAbuse=acknowledge_abuse, + supportsAllDrives=True, + ) + try: + return consume(self._WrapRequest(request), None) + except ApiRequestError as error: + if ( + error.error["code"] != 403 + or error.GetField("reason") != "fileNotDownloadable" + ): + raise + + export_type = mimetype or "text/plain" + request = files.export_media(fileId=file_id, mimeType=export_type) + return consume(self._WrapRequest(request), export_type) + @LoadAuth def FetchMetadata(self, fields=None, fetch_all=False): """Download file's metadata from id using Files.get(). diff --git a/pydrive2/fs/spec.py b/pydrive2/fs/spec.py index bdf1927..4901f1b 100644 --- a/pydrive2/fs/spec.py +++ b/pydrive2/fs/spec.py @@ -309,7 +309,11 @@ def _gdrive_shared_drive_id(self, item_id): return item.get("driveId", None) def _gdrive_list(self, query): - param = {"q": query, "pageSize": 1000} + param = { + "q": query, + "pageSize": 1000, + "fields": "nextPageToken,files(id,name,mimeType,parents,size,md5Checksum)", + } param.update(self._list_params) file_list = self.client.ListFile(param) @@ -424,7 +428,7 @@ def info(self, path): bucket, base = self.split_path(path) item_id = self._get_item_id(path) gdrive_file = self.client.CreateFile({"id": item_id}) - gdrive_file.FetchMetadata() + gdrive_file.FetchMetadata(fields="mimeType,size,md5Checksum") metadata = {"name": posixpath.join(bucket, base.rstrip("/"))} if gdrive_file["mimeType"] == FOLDER_MIME_TYPE: @@ -547,13 +551,15 @@ def _gdrive_upload_fobj(self, title, parent_id, stream, callback=None): return item def cp_file(self, lpath, rpath, **kwargs): - """In-memory streamed copy""" - with self.open(lpath) as stream: - # IterStream objects doesn't support full-length - # seek() calls, so we have to wrap the data with - # an external buffer. - buffer = io.BytesIO(stream.read()) - self.upload_fobj(buffer, rpath) + """Ask Drive to copy the file directly into the destination folder.""" + source_id = self._get_item_id(lpath) + parent_id = self._get_item_id(self._parent(rpath), create=True) + source = self.client.CreateFile({"id": source_id}) + source.Copy( + target_folder={"id": parent_id}, + new_title=posixpath.basename(rpath.rstrip("/")), + param={"fields": "id"}, + ) @_gdrive_retry def mv(self, path1, path2, maxdepth=None, **kwargs): @@ -602,12 +608,15 @@ def _gdrive_get_file(self, item_id, rpath, callback=None, block_size=None): extra_args["chunksize"] = block_size if callback: + reported_size = None - def cb(value, _): + def cb(value, total): + nonlocal reported_size + if total is not None and total != reported_size: + callback.set_size(total) + reported_size = total callback.absolute_update(value) - gdrive_file.FetchMetadata(fields="size") - callback.set_size(int(gdrive_file.get("size"))) extra_args["callback"] = cb gdrive_file.GetContentFile(rpath, **extra_args) diff --git a/pydrive2/test/test_drive_v3.py b/pydrive2/test/test_drive_v3.py index f7c62c8..e98e584 100644 --- a/pydrive2/test/test_drive_v3.py +++ b/pydrive2/test/test_drive_v3.py @@ -1,11 +1,15 @@ import json +from codecs import BOM_UTF8 from urllib.parse import parse_qs, urlsplit +import pytest from google.oauth2.credentials import Credentials from googleapiclient.http import HttpMockSequence from pydrive2.auth import GoogleAuth from pydrive2.drive import GoogleDrive +from pydrive2.files import ApiRequestError, GoogleDriveFile +from pydrive2.fs import GDriveFileSystem def mock_drive(*responses): @@ -129,3 +133,99 @@ def test_v3_permissions_and_revisions(): ] assert parse_qs(urlsplit(requests[2][0]).query)["pageToken"] == ["p2"] assert parse_qs(urlsplit(requests[5][0]).query)["pageToken"] == ["r2"] + + +@pytest.mark.parametrize("stream", [True, False]) +@pytest.mark.parametrize("known_type", [True, False]) +def test_workspace_export_requests(stream, known_type, tmp_path): + body = BOM_UTF8 + b"text" + responses = [({"status": "200", "content-length": str(len(body))}, body)] + if not known_type: + responses.insert( + 0, + ( + {"status": "403"}, + json.dumps( + { + "error": { + "code": 403, + "errors": [{"reason": "fileNotDownloadable"}], + } + } + ), + ), + ) + drive, requests = mock_drive(*responses) + metadata = {"id": "doc", "resourceKey": "key"} + if known_type: + metadata["mimeType"] = "application/vnd.google-apps.document" + file = GoogleDriveFile(drive.auth, metadata, uploaded=True) + + if stream: + assert b"".join(file.GetContentIOBuffer(remove_bom=True)) == b"text" + else: + path = tmp_path / "document" + file.GetContentFile(path, remove_bom=True) + assert path.read_bytes() == b"text" + + assert len(requests) == (1 if known_type else 2) + assert urlsplit(requests[-1][0]).path.endswith("/doc/export") + assert all( + r[3]["X-Goog-Drive-Resource-Keys"] == "doc/key" for r in requests + ) + + +def test_download_permission_error_does_not_trigger_export(): + drive, requests = mock_drive( + ( + {"status": "403"}, + json.dumps( + { + "error": { + "code": 403, + "errors": [{"reason": "insufficientPermissions"}], + } + } + ), + ) + ) + with pytest.raises(ApiRequestError): + drive.CreateFile({"id": "private"}).GetContentIOBuffer() + assert len(requests) == 1 + + +def test_filesystem_copy_uses_drive_and_creates_parents(mocker): + drive, requests = mock_drive(response({"id": "new-copy"})) + fs = GDriveFileSystem( + "root", google_auth=drive.auth, skip_instance_cache=True + ) + lookup = mocker.patch.object( + fs, "_get_item_id", side_effect=["source", "dest-dir"] + ) + fs.cp_file("root/payload", "root/new/nested/copy") + + assert lookup.call_args_list == [ + mocker.call("root/payload"), + mocker.call("root/new/nested", create=True), + ] + ((url, method, body, _),) = requests + assert urlsplit(url).path == "/drive/v3/files/source/copy" + assert method == "POST" + assert json.loads(body) == {"name": "copy", "parents": ["dest-dir"]} + + +def test_filesystem_download_progress_uses_media_response(mocker, tmp_path): + drive, requests = mock_drive( + ({"status": "200", "content-length": "3"}, b"xyz") + ) + fs = GDriveFileSystem( + "root", google_auth=drive.auth, skip_instance_cache=True + ) + mocker.patch.object(fs, "_get_item_id", return_value="file") + callback = mocker.Mock() + fs.get_file("root/payload", tmp_path / "payload", callback=callback) + callback.set_size.assert_called_once_with(3) + callback.absolute_update.assert_called_once_with(3) + assert (tmp_path / "payload").read_bytes() == b"xyz" + assert len(requests) == 1 + assert parse_qs(urlsplit(requests[0][0]).query)["alt"] == ["media"] diff --git a/pydrive2/test/test_fs.py b/pydrive2/test/test_fs.py index 73054d5..e013fa8 100644 --- a/pydrive2/test/test_fs.py +++ b/pydrive2/test/test_fs.py @@ -131,6 +131,15 @@ def test_move(fs, remote_dir): assert initial_info == secondary_info +def test_copy_file_to_new_directory(fs, remote_dir): + source = remote_dir + "/source" + target = remote_dir + "/nested/copy" + fs.pipe_file(source, b"payload") + fs.cp_file(source, target) + assert fs.cat_file(target) == b"payload" + assert fs.cat_file(source) == b"payload" + + def test_rm(fs, remote_dir): fs.touch(remote_dir + "/a.txt") fs.rm(remote_dir + "/a.txt") From b060392d85cd719b004d8d8e0ebe8cabf1510d13 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sat, 21 Nov 2026 14:15:06 -0800 Subject: [PATCH 2/2] Release consumed download chunks and export known Workspace files directly --- docs/filemanagement.rst | 4 +- pydrive2/files.py | 119 +++++-------------------------- pydrive2/test/test_file.py | 56 +-------------- pydrive2/test/test_files_unit.py | 73 ++++++++++++++++++- 4 files changed, 95 insertions(+), 157 deletions(-) diff --git a/docs/filemanagement.rst b/docs/filemanagement.rst index 2134789..3186f38 100644 --- a/docs/filemanagement.rst +++ b/docs/filemanagement.rst @@ -340,7 +340,9 @@ Download file content Use :meth:`~pydrive2.files.GoogleDriveFile.GetContentFile` to download to disk or :meth:`~pydrive2.files.GoogleDriveFile.GetContentString` to read -the content as text. +the content as text. Disk downloads open the destination after the first +successful read. If a later read fails, the file contains the chunks +already written. Sample code continues from above: diff --git a/pydrive2/files.py b/pydrive2/files.py index 56f5f60..eb3aca0 100644 --- a/pydrive2/files.py +++ b/pydrive2/files.py @@ -34,7 +34,6 @@ } ) -BLOCK_SIZE = 1024 # Usage: MIME_TYPE_TO_BOM['']['']. MIME_TYPE_TO_BOM = { "application/vnd.google-apps.document": {"text/plain": "\ufeff".encode()} @@ -140,19 +139,15 @@ def _GetList(self): class IoBuffer: """Lightweight retention of one chunk.""" - def __init__(self, encoding): - self.encoding = encoding - self.chunk = None + def __init__(self): + self.chunk = b"" def write(self, chunk): self.chunk = chunk def read(self): - return ( - self.chunk.decode(self.encoding) - if self.chunk and self.encoding - else self.chunk - ) + chunk, self.chunk = self.chunk, b"" + return chunk class MediaIoReadable: @@ -172,7 +167,7 @@ def __init__( :raises: ApiRequestError """ self.done = False - self._fd = IoBuffer(None) + self._fd = IoBuffer() self.downloader = MediaIoBaseDownload( self._fd, request, chunksize=chunksize ) @@ -191,7 +186,7 @@ def _next_chunk(self): raise ApiRequestError(error) from error self.size = status.total_size self.progress = status.resumable_progress - return self._fd.read() or b"" + return self._fd.read() def _read_chunk(self): chunk = self._next_chunk() @@ -218,14 +213,7 @@ def read(self): return self._read_chunk() def __iter__(self): - """ - :raises: ApiRequestError - """ - while True: - chunk = self.read() - if chunk is None: - break - yield chunk + return iter(self.read, None) def __len__(self): return self.size @@ -361,6 +349,9 @@ def GetContentFile( ): """Save content of this file as a local file. + The destination is opened after the first successful read. If a later + read fails, the destination contains the chunks written so far. + :param filename: name of the file to write to. :type filename: str :param mimetype: mimeType of the file. @@ -386,6 +377,8 @@ def GetContentFile( with open(filename, "wb") as fd: for chunk in stream: fd.write(chunk) + # Release the response body before the next HTTP request. + del chunk if callback: callback(stream.progress, stream.size) @@ -426,7 +419,7 @@ def open_stream(request, content_type): file_id = self._FileId() files = self.auth.service.files() - source_type = self.metadata.get("mimeType") or self.get("mimeType") + source_type = self._SourceMimeType() if source_type not in WORKSPACE_EXPORT_TYPES: request = files.get_media( fileId=file_id, @@ -473,10 +466,11 @@ def FetchMetadata(self, fields=None, fetch_all=False): self.uploaded = True self.UpdateMetadata(metadata) - @LoadMetadata def FetchContent(self, mimetype=None, remove_bom=False): """Download file content or a Workspace export into memory.""" - if self.get("mimeType") in { + if not self.uploaded and not self._SourceMimeType(): + self.FetchMetadata() + if self._SourceMimeType() in { "application/vnd.google-apps.folder", "application/vnd.google-apps.shortcut", }: @@ -708,6 +702,9 @@ def _ListChildren(self, resource, key, **params): self[key] = self.metadata[key] = items return items + def _SourceMimeType(self): + return self.metadata.get("mimeType") or self.get("mimeType") + def _FileId(self): file_id = self.metadata.get("id") or self.get("id") if not file_id: @@ -763,81 +760,3 @@ def _GetBOM(mimetype): for bom in MIME_TYPE_TO_BOM.values(): if mimetype in bom: return bom[mimetype] - - @staticmethod - def _RemovePrefix(file_object, prefix, block_size=BLOCK_SIZE): - """Deletes passed prefix by shifting content of passed file object by to - the left. Operation is in-place. - - Args: - file_object (obj): The file object to manipulate. - prefix (str): The prefix to insert. - block_size (int): The size of the blocks which are moved one at a time. - """ - prefix_length = len(prefix) - # Detect if prefix exists in file. - content_start = file_object.read(prefix_length) - - if content_start == prefix: - # Shift content left by prefix length, by copying 1KiB at a time. - block_to_write = file_object.read(block_size) - current_block_length = len(block_to_write) - - # Read and write location in separate variables for simplicity. - read_location = prefix_length + current_block_length - write_location = 0 - - while current_block_length > 0: - # Write next block. - file_object.seek(write_location) - file_object.write(block_to_write) - # Set write location to the next block. - write_location += len(block_to_write) - - # Read next block of input. - file_object.seek(read_location) - block_to_write = file_object.read(block_size) - # Update the current block length and read_location. - current_block_length = len(block_to_write) - read_location += current_block_length - - # Truncate the file to its, now shorter, length. - file_object.truncate(read_location - prefix_length) - - @staticmethod - def _InsertPrefix(file_object, prefix, block_size=BLOCK_SIZE): - """Inserts the passed prefix in the beginning of the file, operation is - in-place. - - Args: - file_object (obj): The file object to manipulate. - prefix (str): The prefix to insert. - """ - # Read the first two blocks. - first_block = file_object.read(block_size) - second_block = file_object.read(block_size) - # Pointer to the first byte of the next block to be read. - read_location = block_size * 2 - - # Write BOM. - file_object.seek(0) - file_object.write(prefix) - # {read|write}_location separated for readability. - write_location = len(prefix) - - # Write and read block alternatingly. - while len(first_block): - # Write first block. - file_object.seek(write_location) - file_object.write(first_block) - # Increment write_location. - write_location += block_size - - # Move second block into first variable. - first_block = second_block - - # Read in the next block. - file_object.seek(read_location) - second_block = file_object.read(block_size) - # Increment read_location. - read_location += block_size diff --git a/pydrive2/test/test_file.py b/pydrive2/test/test_file.py index c3c6a9b..23da9b3 100644 --- a/pydrive2/test/test_file.py +++ b/pydrive2/test/test_file.py @@ -4,7 +4,6 @@ import sys import unittest from concurrent.futures import ThreadPoolExecutor, as_completed -from io import BytesIO from pathlib import Path from tempfile import mkdtemp from time import time @@ -17,7 +16,7 @@ from pydrive2.auth import GoogleAuth from pydrive2.drive import GoogleDrive -from pydrive2.files import ApiRequestError, GoogleDriveFile +from pydrive2.files import ApiRequestError from pydrive2.test import test_util from pydrive2.test.test_util import ( create_file, @@ -1089,58 +1088,5 @@ def DeleteUploadedFiles(self, drive, ids): pydrive_retry(tmp_file.Delete) -class FilePrefixTest(unittest.TestCase): - def test_InsertPrefix(self): - # Create BytesIO. - file_obj = BytesIO(b"abc") - original_length = len(file_obj.getvalue()) - char_to_insert = "\ufeff".encode() - - # Insert the prefix. - GoogleDriveFile._InsertPrefix(file_obj, char_to_insert) - modified_length = len(file_obj.getvalue()) - self.assertGreater(modified_length, original_length) - self.assertEqual(file_obj.getvalue(), "\ufeffabc".encode()) - - def test_InsertPrefixLarge(self): - # Create BytesIO. - test_content = "abc" * 800 - file_obj = BytesIO(test_content.encode("utf-8")) - original_length = len(file_obj.getvalue()) - char_to_insert = "\ufeff".encode() - - # Insert the prefix. - GoogleDriveFile._InsertPrefix(file_obj, char_to_insert) - modified_length = len(file_obj.getvalue()) - self.assertGreater(modified_length, original_length) - expected_content = "\ufeff" + test_content - self.assertEqual(file_obj.getvalue(), expected_content.encode("utf8")) - - def test_RemovePrefix(self): - # Create BytesIO. - file_obj = BytesIO("\ufeffabc".encode()) - original_length = len(file_obj.getvalue()) - char_to_remove = "\ufeff".encode() - - # Insert the prefix. - GoogleDriveFile._RemovePrefix(file_obj, char_to_remove) - modified_length = len(file_obj.getvalue()) - self.assertLess(modified_length, original_length) - self.assertEqual(file_obj.getvalue(), b"abc") - - def test_RemovePrefixLarge(self): - # Create BytesIO. - test_content = "\ufeff" + "abc" * 800 - file_obj = BytesIO(test_content.encode("utf8")) - original_length = len(file_obj.getvalue()) - char_to_remove = "\ufeff".encode() - - # Insert the prefix. - GoogleDriveFile._RemovePrefix(file_obj, char_to_remove) - modified_length = len(file_obj.getvalue()) - self.assertLess(modified_length, original_length) - self.assertEqual(file_obj.getvalue(), test_content[1:].encode("utf8")) - - if __name__ == "__main__": unittest.main() diff --git a/pydrive2/test/test_files_unit.py b/pydrive2/test/test_files_unit.py index 1ccdfaa..72c5351 100644 --- a/pydrive2/test/test_files_unit.py +++ b/pydrive2/test/test_files_unit.py @@ -1,4 +1,6 @@ +import gc import json +import weakref from codecs import BOM_UTF8 from urllib.parse import urlsplit @@ -278,6 +280,33 @@ def read_disk(file, tmp_path, **kwargs): return dest.read_bytes() +def read_string(file, tmp_path, **kwargs): + return file.GetContentString(**kwargs).encode() + + +@pytest.mark.parametrize("read", [read_disk, read_stream, read_string]) +@pytest.mark.parametrize( + ("source", "target"), + [ + ("document", "text/plain"), + ("spreadsheet", "text/csv"), + ("presentation", "text/plain"), + ("drawing", "image/svg+xml"), + ], +) +def test_supplied_workspace_type_goes_directly_to_export( + mock_drive, tmp_path, read, source, target +): + drive, requests = mock_drive(({"status": "200"}, b"export")) + file = drive.CreateFile( + {"id": "doc", "mimeType": f"application/vnd.google-apps.{source}"} + ) + assert read(file, tmp_path, mimetype=target) == b"export" + (export,) = requests + assert urlsplit(export[0]).path == "/drive/v3/files/doc/export" + assert query(export)["mimeType"] == [target] + + @pytest.mark.parametrize("read", [read_stream, read_disk]) @pytest.mark.parametrize("known_type", [True, False], ids=["known", "id-only"]) def test_workspace_export_requests(mock_drive, read, known_type, tmp_path): @@ -339,7 +368,9 @@ def test_stream_decodes_utf8_across_chunks(mock_drive): ({"status": "503"}, "unavailable"), ], ) -def test_download_error_preserves_existing_file(mock_drive, tmp_path, denied): +def test_first_download_error_preserves_existing_file( + mock_drive, tmp_path, denied +): drive, requests = mock_drive(denied) dest = tmp_path / "existing" dest.write_bytes(b"previous contents") @@ -348,3 +379,43 @@ def test_download_error_preserves_existing_file(mock_drive, tmp_path, denied): assert caught.value.error["code"] == int(denied[0]["status"]) assert dest.read_bytes() == b"previous contents" assert len(requests) == 1 + + +def test_later_download_error_leaves_completed_chunks(mock_drive, tmp_path): + drive, _ = mock_drive( + ({"status": "206", "content-range": "bytes 0-1/4"}, b"ab"), + error_response(503, "backendError"), + ) + dest = tmp_path / "existing" + dest.write_bytes(b"previous contents") + with pytest.raises(ApiRequestError): + drive.CreateFile({"id": "file"}).GetContentFile(dest, chunksize=2) + assert dest.read_bytes() == b"ab" + + +def test_disk_download_releases_each_chunk_before_requesting_the_next( + mock_drive, tmp_path, monkeypatch +): + class Chunk(bytearray): + """A response body whose lifetime can be observed with a weak reference.""" + + first = Chunk(b"ab") + previous = weakref.ref(first) + drive, _ = mock_drive( + ({"status": "206", "content-range": "bytes 0-1/4"}, first), + ({"status": "206", "content-range": "bytes 2-3/4"}, b"cd"), + ) + del first + transport = drive.auth.thread_local.http + request = transport.request + + def check_request(uri, method, **kwargs): + if kwargs["headers"]["range"] == "bytes=2-3": + gc.collect() + assert previous() is None + return request(uri, method, **kwargs) + + monkeypatch.setattr(transport, "request", check_request) + dest = tmp_path / "download" + drive.CreateFile({"id": "file"}).GetContentFile(dest, chunksize=2) + assert dest.read_bytes() == b"abcd"