diff --git a/docs/filemanagement.rst b/docs/filemanagement.rst index 2134789..3186f38 100644 --- a/docs/filemanagement.rst +++ b/docs/filemanagement.rst @@ -340,7 +340,9 @@ Download file content Use :meth:`~pydrive2.files.GoogleDriveFile.GetContentFile` to download to disk or :meth:`~pydrive2.files.GoogleDriveFile.GetContentString` to read -the content as text. +the content as text. Disk downloads open the destination after the first +successful read. If a later read fails, the file contains the chunks +already written. Sample code continues from above: diff --git a/pydrive2/files.py b/pydrive2/files.py index 221999d..eb3aca0 100644 --- a/pydrive2/files.py +++ b/pydrive2/files.py @@ -1,3 +1,4 @@ +import codecs import io import json import mimetypes @@ -33,7 +34,6 @@ } ) -BLOCK_SIZE = 1024 # Usage: MIME_TYPE_TO_BOM['']['']. MIME_TYPE_TO_BOM = { "application/vnd.google-apps.document": {"text/plain": "\ufeff".encode()} @@ -139,19 +139,15 @@ def _GetList(self): class IoBuffer: """Lightweight retention of one chunk.""" - def __init__(self, encoding): - self.encoding = encoding - self.chunk = None + def __init__(self): + self.chunk = b"" def write(self, chunk): self.chunk = chunk def read(self): - return ( - self.chunk.decode(self.encoding) - if self.chunk and self.encoding - else self.chunk - ) + chunk, self.chunk = self.chunk, b"" + return chunk class MediaIoReadable: @@ -171,46 +167,53 @@ def __init__( :raises: ApiRequestError """ self.done = False - self._fd = IoBuffer(encoding) + self._fd = IoBuffer() self.downloader = MediaIoBaseDownload( self._fd, request, chunksize=chunksize ) self.size = None - self._pre_buffer = False - if pre_buffer: - self.read() - if remove_prefix: - chunk = io.BytesIO(self._fd.chunk or b"") - GoogleDriveFile._RemovePrefix(chunk, remove_prefix) - self._fd.chunk = chunk.getvalue() - self._pre_buffer = True + self.progress = 0 + self._decoder = ( + codecs.getincrementaldecoder(encoding)() if encoding else None + ) + self._prefix = remove_prefix + self._buffer = self._read_chunk() if pre_buffer else None - def read(self): - """ - :returns: bytes or str -- chunk (or None if done) - :raises: ApiRequestError - """ - if self._pre_buffer: - self._pre_buffer = False - return self._fd.read() - if self.done: - return None + def _next_chunk(self): try: status, self.done = self.downloader.next_chunk() - self.size = status.total_size except errors.HttpError as error: - raise ApiRequestError(error) + raise ApiRequestError(error) from error + self.size = status.total_size + self.progress = status.resumable_progress return self._fd.read() + def _read_chunk(self): + chunk = self._next_chunk() + if self._prefix: + while ( + not self.done + and len(chunk) < len(self._prefix) + and self._prefix.startswith(chunk) + ): + chunk += self._next_chunk() + chunk = chunk.removeprefix(self._prefix) + self._prefix = b"" + if self._decoder: + return self._decoder.decode(chunk, final=self.done) + return chunk + + def read(self): + """Return the next chunk, or None at end of stream.""" + if self._buffer is not None: + chunk, self._buffer = self._buffer, None + return chunk + if self.done: + return None + return self._read_chunk() + def __iter__(self): - """ - :raises: ApiRequestError - """ - while True: - chunk = self.read() - if chunk is None: - break - yield chunk + return iter(self.read, None) def __len__(self): return self.size @@ -335,7 +338,6 @@ def GetContentString( self.FetchContent(mimetype, remove_bom) return self.content.getvalue().decode(encoding) - @LoadAuth def GetContentFile( self, filename, @@ -347,6 +349,9 @@ def GetContentFile( ): """Save content of this file as a local file. + The destination is opened after the first successful read. If a later + read fails, the destination contains the chunks written so far. + :param filename: name of the file to write to. :type filename: str :param mimetype: mimeType of the file. @@ -362,55 +367,20 @@ def GetContentFile( :type acknowledge_abuse: bool :raises: ApiRequestError, FileNotUploadedError """ - files = self.auth.service.files() - file_id = self._FileId() - - def download(fd, request): - downloader = MediaIoBaseDownload( - fd, self._WrapRequest(request), chunksize=chunksize - ) - done = False - while done is False: - status, done = downloader.next_chunk() + stream = self.GetContentIOBuffer( + mimetype=mimetype, + remove_bom=remove_bom, + chunksize=chunksize, + acknowledge_abuse=acknowledge_abuse, + ) + # The pre-buffered first response determines whether the file is readable. + with open(filename, "wb") as fd: + for chunk in stream: + fd.write(chunk) + # Release the response body before the next HTTP request. + del chunk if callback: - callback(status.resumable_progress, status.total_size) - - with open(filename, mode="w+b") as fd: - # Should use files.export_media instead of files.get_media if - # metadata["mimeType"].startswith("application/vnd.google-apps."). - # But that would first require a slow call to FetchMetadata(). - # We prefer to try-except for speed. - try: - download( - fd, - files.get_media( - fileId=file_id, - acknowledgeAbuse=acknowledge_abuse, - supportsAllDrives=True, - ), - ) - except errors.HttpError as error: - exc = ApiRequestError(error) - if ( - exc.error["code"] != 403 - or exc.GetField("reason") != "fileNotDownloadable" - ): - raise exc - mimetype = mimetype or "text/plain" - fd.seek(0) # just in case `download()` modified `fd` - try: - download( - fd, - files.export_media(fileId=file_id, mimeType=mimetype), - ) - except errors.HttpError as error: - raise ApiRequestError(error) - - if mimetype == "text/plain" and remove_bom: - fd.seek(0) - bom = self._GetBOM(mimetype) - if bom: - self._RemovePrefix(fd, bom) + callback(stream.progress, stream.size) @LoadAuth def GetContentIOBuffer( @@ -449,7 +419,7 @@ def open_stream(request, content_type): file_id = self._FileId() files = self.auth.service.files() - source_type = self.metadata.get("mimeType") or self.get("mimeType") + source_type = self._SourceMimeType() if source_type not in WORKSPACE_EXPORT_TYPES: request = files.get_media( fileId=file_id, @@ -496,10 +466,11 @@ def FetchMetadata(self, fields=None, fetch_all=False): self.uploaded = True self.UpdateMetadata(metadata) - @LoadMetadata def FetchContent(self, mimetype=None, remove_bom=False): """Download file content or a Workspace export into memory.""" - if self.get("mimeType") in { + if not self.uploaded and not self._SourceMimeType(): + self.FetchMetadata() + if self._SourceMimeType() in { "application/vnd.google-apps.folder", "application/vnd.google-apps.shortcut", }: @@ -731,6 +702,9 @@ def _ListChildren(self, resource, key, **params): self[key] = self.metadata[key] = items return items + def _SourceMimeType(self): + return self.metadata.get("mimeType") or self.get("mimeType") + def _FileId(self): file_id = self.metadata.get("id") or self.get("id") if not file_id: @@ -786,81 +760,3 @@ def _GetBOM(mimetype): for bom in MIME_TYPE_TO_BOM.values(): if mimetype in bom: return bom[mimetype] - - @staticmethod - def _RemovePrefix(file_object, prefix, block_size=BLOCK_SIZE): - """Deletes passed prefix by shifting content of passed file object by to - the left. Operation is in-place. - - Args: - file_object (obj): The file object to manipulate. - prefix (str): The prefix to insert. - block_size (int): The size of the blocks which are moved one at a time. - """ - prefix_length = len(prefix) - # Detect if prefix exists in file. - content_start = file_object.read(prefix_length) - - if content_start == prefix: - # Shift content left by prefix length, by copying 1KiB at a time. - block_to_write = file_object.read(block_size) - current_block_length = len(block_to_write) - - # Read and write location in separate variables for simplicity. - read_location = prefix_length + current_block_length - write_location = 0 - - while current_block_length > 0: - # Write next block. - file_object.seek(write_location) - file_object.write(block_to_write) - # Set write location to the next block. - write_location += len(block_to_write) - - # Read next block of input. - file_object.seek(read_location) - block_to_write = file_object.read(block_size) - # Update the current block length and read_location. - current_block_length = len(block_to_write) - read_location += current_block_length - - # Truncate the file to its, now shorter, length. - file_object.truncate(read_location - prefix_length) - - @staticmethod - def _InsertPrefix(file_object, prefix, block_size=BLOCK_SIZE): - """Inserts the passed prefix in the beginning of the file, operation is - in-place. - - Args: - file_object (obj): The file object to manipulate. - prefix (str): The prefix to insert. - """ - # Read the first two blocks. - first_block = file_object.read(block_size) - second_block = file_object.read(block_size) - # Pointer to the first byte of the next block to be read. - read_location = block_size * 2 - - # Write BOM. - file_object.seek(0) - file_object.write(prefix) - # {read|write}_location separated for readability. - write_location = len(prefix) - - # Write and read block alternatingly. - while len(first_block): - # Write first block. - file_object.seek(write_location) - file_object.write(first_block) - # Increment write_location. - write_location += block_size - - # Move second block into first variable. - first_block = second_block - - # Read in the next block. - file_object.seek(read_location) - second_block = file_object.read(block_size) - # Increment read_location. - read_location += block_size diff --git a/pydrive2/fs/spec.py b/pydrive2/fs/spec.py index 86cc862..6166662 100644 --- a/pydrive2/fs/spec.py +++ b/pydrive2/fs/spec.py @@ -28,6 +28,8 @@ logger = logging.getLogger(__name__) FOLDER_MIME_TYPE = "application/vnd.google-apps.folder" +_INFO_FIELDS = "mimeType,size,md5Checksum" +_LIST_FIELDS = f"nextPageToken,files(id,name,parents,{_INFO_FIELDS})" _PARENT_QUERY_BATCH_SIZE = 50 COMMON_SETTINGS = { @@ -334,7 +336,11 @@ def _gdrive_shared_drive_id(self, item_id): return item.get("driveId", None) def _gdrive_list(self, query): - param = {"q": query, "pageSize": 1000} + param = { + "q": query, + "pageSize": 1000, + "fields": _LIST_FIELDS, + } param.update(self._list_params) file_list = self.client.ListFile(param) @@ -454,7 +460,7 @@ def info(self, path): bucket, base = self.split_path(path) item_id = self._get_item_id(path) gdrive_file = self.client.CreateFile({"id": item_id}) - gdrive_file.FetchMetadata() + gdrive_file.FetchMetadata(fields=_INFO_FIELDS) return _file_entry( posixpath.join(bucket, base.rstrip("/")), gdrive_file @@ -557,13 +563,20 @@ def _gdrive_upload_fobj(self, name, parent_id, stream, callback=None): return item def cp_file(self, lpath, rpath, **kwargs): - """In-memory streamed copy""" - with self.open(lpath) as stream: - # IterStream objects doesn't support full-length - # seek() calls, so we have to wrap the data with - # an external buffer. - buffer = io.BytesIO(stream.read()) - self.upload_fobj(buffer, rpath) + """Ask Drive to copy the file directly into the destination folder.""" + source_id = self._get_item_id(lpath) + parent_id = self._get_item_id(self._parent(rpath), create=True) + name = posixpath.basename(rpath.rstrip("/")) + self._gdrive_copy_file(source_id, parent_id, name) + + @_gdrive_retry + def _gdrive_copy_file(self, source_id, parent_id, name): + source = self.client.CreateFile({"id": source_id}) + source.Copy( + target_folder={"id": parent_id}, + new_title=name, + param={"fields": "id"}, + ) @_gdrive_retry def mv(self, path1, path2, maxdepth=None, **kwargs): @@ -612,12 +625,15 @@ def _gdrive_get_file(self, item_id, rpath, callback=None, block_size=None): extra_args["chunksize"] = block_size if callback: + reported_size = None - def cb(value, _): + def cb(value, total): + nonlocal reported_size + if total is not None and total != reported_size: + callback.set_size(total) + reported_size = total callback.absolute_update(value) - gdrive_file.FetchMetadata(fields="size") - callback.set_size(int(gdrive_file.get("size"))) extra_args["callback"] = cb gdrive_file.GetContentFile(rpath, **extra_args) diff --git a/pydrive2/test/test_file.py b/pydrive2/test/test_file.py index b28a193..3e9fa43 100644 --- a/pydrive2/test/test_file.py +++ b/pydrive2/test/test_file.py @@ -4,7 +4,6 @@ import sys import unittest from concurrent.futures import ThreadPoolExecutor, as_completed -from io import BytesIO from pathlib import Path from tempfile import mkdtemp from time import time @@ -17,7 +16,7 @@ from pydrive2.auth import GoogleAuth from pydrive2.drive import GoogleDrive -from pydrive2.files import ApiRequestError, GoogleDriveFile +from pydrive2.files import ApiRequestError from pydrive2.test import test_util from pydrive2.test.test_util import ( create_file, @@ -1088,58 +1087,5 @@ def DeleteUploadedFiles(self, drive, ids): pydrive_retry(tmp_file.Delete) -class FilePrefixTest(unittest.TestCase): - def test_InsertPrefix(self): - # Create BytesIO. - file_obj = BytesIO(b"abc") - original_length = len(file_obj.getvalue()) - char_to_insert = "\ufeff".encode() - - # Insert the prefix. - GoogleDriveFile._InsertPrefix(file_obj, char_to_insert) - modified_length = len(file_obj.getvalue()) - self.assertGreater(modified_length, original_length) - self.assertEqual(file_obj.getvalue(), "\ufeffabc".encode()) - - def test_InsertPrefixLarge(self): - # Create BytesIO. - test_content = "abc" * 800 - file_obj = BytesIO(test_content.encode("utf-8")) - original_length = len(file_obj.getvalue()) - char_to_insert = "\ufeff".encode() - - # Insert the prefix. - GoogleDriveFile._InsertPrefix(file_obj, char_to_insert) - modified_length = len(file_obj.getvalue()) - self.assertGreater(modified_length, original_length) - expected_content = "\ufeff" + test_content - self.assertEqual(file_obj.getvalue(), expected_content.encode("utf8")) - - def test_RemovePrefix(self): - # Create BytesIO. - file_obj = BytesIO("\ufeffabc".encode()) - original_length = len(file_obj.getvalue()) - char_to_remove = "\ufeff".encode() - - # Insert the prefix. - GoogleDriveFile._RemovePrefix(file_obj, char_to_remove) - modified_length = len(file_obj.getvalue()) - self.assertLess(modified_length, original_length) - self.assertEqual(file_obj.getvalue(), b"abc") - - def test_RemovePrefixLarge(self): - # Create BytesIO. - test_content = "\ufeff" + "abc" * 800 - file_obj = BytesIO(test_content.encode("utf8")) - original_length = len(file_obj.getvalue()) - char_to_remove = "\ufeff".encode() - - # Insert the prefix. - GoogleDriveFile._RemovePrefix(file_obj, char_to_remove) - modified_length = len(file_obj.getvalue()) - self.assertLess(modified_length, original_length) - self.assertEqual(file_obj.getvalue(), test_content[1:].encode("utf8")) - - if __name__ == "__main__": unittest.main() diff --git a/pydrive2/test/test_files_unit.py b/pydrive2/test/test_files_unit.py index 1752338..72c5351 100644 --- a/pydrive2/test/test_files_unit.py +++ b/pydrive2/test/test_files_unit.py @@ -1,4 +1,7 @@ +import gc import json +import weakref +from codecs import BOM_UTF8 from urllib.parse import urlsplit import pytest @@ -265,3 +268,154 @@ def test_child_list_errors_retain_drive_reason(mock_drive, operation): with pytest.raises(ApiRequestError) as caught: getattr(drive.CreateFile({"id": "file"}), operation)() assert caught.value.GetField("reason") == "rateLimitExceeded" + + +def read_stream(file, tmp_path, **kwargs): + return b"".join(file.GetContentIOBuffer(**kwargs)) + + +def read_disk(file, tmp_path, **kwargs): + dest = tmp_path / "document" + file.GetContentFile(dest, **kwargs) + return dest.read_bytes() + + +def read_string(file, tmp_path, **kwargs): + return file.GetContentString(**kwargs).encode() + + +@pytest.mark.parametrize("read", [read_disk, read_stream, read_string]) +@pytest.mark.parametrize( + ("source", "target"), + [ + ("document", "text/plain"), + ("spreadsheet", "text/csv"), + ("presentation", "text/plain"), + ("drawing", "image/svg+xml"), + ], +) +def test_supplied_workspace_type_goes_directly_to_export( + mock_drive, tmp_path, read, source, target +): + drive, requests = mock_drive(({"status": "200"}, b"export")) + file = drive.CreateFile( + {"id": "doc", "mimeType": f"application/vnd.google-apps.{source}"} + ) + assert read(file, tmp_path, mimetype=target) == b"export" + (export,) = requests + assert urlsplit(export[0]).path == "/drive/v3/files/doc/export" + assert query(export)["mimeType"] == [target] + + +@pytest.mark.parametrize("read", [read_stream, read_disk]) +@pytest.mark.parametrize("known_type", [True, False], ids=["known", "id-only"]) +def test_workspace_export_requests(mock_drive, read, known_type, tmp_path): + body = BOM_UTF8 + b"text" + responses = [({"status": "200", "content-length": str(len(body))}, body)] + metadata = {"id": "doc", "resourceKey": "key"} + if known_type: + metadata["mimeType"] = "application/vnd.google-apps.document" + else: + responses.insert(0, error_response(403, "fileNotDownloadable")) + drive, requests = mock_drive(*responses) + assert ( + read(drive.CreateFile(metadata), tmp_path, remove_bom=True) == b"text" + ) + assert len(requests) == (1 if known_type else 2) + assert urlsplit(requests[-1][0]).path.endswith("/doc/export") + assert all( + r[3]["X-Goog-Drive-Resource-Keys"] == "doc/key" for r in requests + ) + + +@pytest.mark.parametrize("read", [read_stream, read_disk]) +def test_plain_text_media_removes_bom_across_chunks( + mock_drive, read, tmp_path +): + drive, _ = mock_drive( + ({"status": "206", "content-range": "bytes 0-1/5"}, BOM_UTF8[:2]), + ( + {"status": "206", "content-range": "bytes 2-3/5"}, + BOM_UTF8[2:] + b"x", + ), + ({"status": "206", "content-range": "bytes 4-4/5"}, b"y"), + ) + file = drive.CreateFile({"id": "blob"}) + assert ( + read( + file, tmp_path, mimetype="text/plain", remove_bom=True, chunksize=2 + ) + == b"xy" + ) + + +def test_stream_decodes_utf8_across_chunks(mock_drive): + drive, _ = mock_drive( + ({"status": "206", "content-range": "bytes 0-0/2"}, b"\xc3"), + ({"status": "206", "content-range": "bytes 1-1/2"}, b"\xa9"), + ) + file = drive.CreateFile({"id": "blob"}) + assert ( + "".join(file.GetContentIOBuffer(encoding="utf-8", chunksize=1)) == "é" + ) + + +@pytest.mark.parametrize( + "denied", + [ + error_response(403, "insufficientFilePermissions"), + ({"status": "403"}, '{"error":{"errors":[]}}'), + ({"status": "503"}, "unavailable"), + ], +) +def test_first_download_error_preserves_existing_file( + mock_drive, tmp_path, denied +): + drive, requests = mock_drive(denied) + dest = tmp_path / "existing" + dest.write_bytes(b"previous contents") + with pytest.raises(ApiRequestError) as caught: + drive.CreateFile({"id": "private"}).GetContentFile(dest) + assert caught.value.error["code"] == int(denied[0]["status"]) + assert dest.read_bytes() == b"previous contents" + assert len(requests) == 1 + + +def test_later_download_error_leaves_completed_chunks(mock_drive, tmp_path): + drive, _ = mock_drive( + ({"status": "206", "content-range": "bytes 0-1/4"}, b"ab"), + error_response(503, "backendError"), + ) + dest = tmp_path / "existing" + dest.write_bytes(b"previous contents") + with pytest.raises(ApiRequestError): + drive.CreateFile({"id": "file"}).GetContentFile(dest, chunksize=2) + assert dest.read_bytes() == b"ab" + + +def test_disk_download_releases_each_chunk_before_requesting_the_next( + mock_drive, tmp_path, monkeypatch +): + class Chunk(bytearray): + """A response body whose lifetime can be observed with a weak reference.""" + + first = Chunk(b"ab") + previous = weakref.ref(first) + drive, _ = mock_drive( + ({"status": "206", "content-range": "bytes 0-1/4"}, first), + ({"status": "206", "content-range": "bytes 2-3/4"}, b"cd"), + ) + del first + transport = drive.auth.thread_local.http + request = transport.request + + def check_request(uri, method, **kwargs): + if kwargs["headers"]["range"] == "bytes=2-3": + gc.collect() + assert previous() is None + return request(uri, method, **kwargs) + + monkeypatch.setattr(transport, "request", check_request) + dest = tmp_path / "download" + drive.CreateFile({"id": "file"}).GetContentFile(dest, chunksize=2) + assert dest.read_bytes() == b"abcd" diff --git a/pydrive2/test/test_fs.py b/pydrive2/test/test_fs.py index 73054d5..e013fa8 100644 --- a/pydrive2/test/test_fs.py +++ b/pydrive2/test/test_fs.py @@ -131,6 +131,15 @@ def test_move(fs, remote_dir): assert initial_info == secondary_info +def test_copy_file_to_new_directory(fs, remote_dir): + source = remote_dir + "/source" + target = remote_dir + "/nested/copy" + fs.pipe_file(source, b"payload") + fs.cp_file(source, target) + assert fs.cat_file(target) == b"payload" + assert fs.cat_file(source) == b"payload" + + def test_rm(fs, remote_dir): fs.touch(remote_dir + "/a.txt") fs.rm(remote_dir + "/a.txt") diff --git a/pydrive2/test/test_fs_unit.py b/pydrive2/test/test_fs_unit.py index 211ed03..a2e1a25 100644 --- a/pydrive2/test/test_fs_unit.py +++ b/pydrive2/test/test_fs_unit.py @@ -1,4 +1,8 @@ +import json +from urllib.parse import urlsplit + import pytest +from fsspec.callbacks import Callback from pydrive2.fs import GDriveFileSystem from pydrive2.fs.spec import FOLDER_MIME_TYPE @@ -85,3 +89,117 @@ def test_shared_drive_delete_permission_error(mock_drive, trash_only): filesystem(drive, "shared", trash_only=trash_only).rm_file( "shared/note" ) + + +def test_copy_creates_missing_directories_and_copies_on_drive(mock_drive): + source = {"name": "payload", "id": "source", "mimeType": "text/plain"} + drive, requests = mock_drive( + response({"files": [source]}), + response({"files": []}), + response({"id": "new"}), + response({"files": []}), + response({"id": "nested"}), + response({"id": "copy"}), + ) + filesystem(drive).cp_file("root/payload", "root/new/nested/copy") + ( + source_lookup, + new_lookup, + new_folder, + nested_lookup, + nested_folder, + copy, + ) = requests + assert [r[1] for r in requests] == [ + "GET", + "GET", + "POST", + "GET", + "POST", + "POST", + ] + assert "name='payload'" in query(source_lookup)["q"][0] + assert "'root' in parents" in query(new_lookup)["q"][0] + assert json.loads(new_folder[2])["parents"] == ["root"] + assert "'new' in parents" in query(nested_lookup)["q"][0] + assert json.loads(nested_folder[2])["parents"] == ["new"] + assert urlsplit(copy[0]).path == "/drive/v3/files/source/copy" + assert json.loads(copy[2]) == {"name": "copy", "parents": ["nested"]} + + +@pytest.mark.parametrize( + ("code", "reason"), [(503, "backendError"), (403, "rateLimitExceeded")] +) +def test_copy_retries_only_copy_request(mock_drive, code, reason): + source = {"name": "payload", "id": "source", "mimeType": "text/plain"} + drive, requests = mock_drive( + response({"files": [source]}), + error_response(code, reason), + response({"id": "copy"}), + ) + filesystem(drive).cp_file("root/payload", "root/copy") + lookup, first, retry = requests + assert lookup[1] == "GET" + assert first[:3] == retry[:3] + assert urlsplit(retry[0]).path.endswith("/source/copy") + + +def test_info_and_listing_request_only_required_fields(mock_drive): + item = { + "name": "payload", + "id": "blob", + "mimeType": "text/plain", + "size": "3", + "md5Checksum": "abc", + } + drive, requests = mock_drive(response({"files": [item]}), response(item)) + assert filesystem(drive).info("root/payload") == { + "name": "root/payload", + "type": "file", + "size": 3, + "checksum": "abc", + } + lookup, get = requests + assert query(lookup)["fields"] == [ + "nextPageToken,files(id,name,parents,mimeType,size,md5Checksum)" + ] + assert query(get)["fields"] == ["mimeType,size,md5Checksum"] + + +@pytest.mark.parametrize( + ("media", "expected"), + [ + ( + [ + ({"status": "206", "content-range": "bytes 0-1/3"}, b"xy"), + ({"status": "206", "content-range": "bytes 2-2/3"}, b"z"), + ], + [(3, 0), (3, 2), (3, 3)], + ), + ([({"status": "200"}, b"xyz")], [(None, 3)]), + ], + ids=["two-chunks", "unknown-total"], +) +def test_download_progress_uses_media_responses( + mock_drive, tmp_path, media, expected +): + source = { + "name": "payload", + "id": "file", + "mimeType": "application/octet-stream", + } + drive, requests = mock_drive(response({"files": [source]}), *media) + updates = [] + callback = Callback( + hooks={"record": lambda size, value: updates.append((size, value))} + ) + dest = tmp_path / "result" + filesystem(drive).get_file( + "root/payload", dest, callback=callback, block_size=2 + ) + lookup, *downloads = requests + assert query(lookup)["q"] + assert len(downloads) == len(media) + assert all(query(request)["alt"] == ["media"] for request in downloads) + assert updates == expected + assert dest.read_bytes() == b"xyz"