diff --git a/README.rst b/README.rst index e3b17fc..0cdc5f4 100644 --- a/README.rst +++ b/README.rst @@ -19,7 +19,7 @@ PyDrive2 -------- *PyDrive2* is a wrapper library of `google-api-python-client`_ that simplifies -many common Google Drive API V2 tasks. It is an actively maintained fork of +many common Google Drive API v3 tasks. It is an actively maintained fork of https://pypi.python.org/pypi/PyDrive. By the authors and maintainers of the `Git for Data `_ - DVC project. @@ -36,7 +36,7 @@ Features of PyDrive2 -------------------- - Simplifies OAuth2.0 into just few lines with flexible settings. -- Wraps `Google Drive API V2`_ into classes of each resource to make your +- Wraps `Google Drive API v3`_ into classes of each resource to make your program more object-oriented. - Helps common operations else than API calls, such as content fetching and pagination control. @@ -83,12 +83,12 @@ efficient way. .. code:: python - file1 = drive.CreateFile({'title': 'Hello.txt'}) + file1 = drive.CreateFile({'name': 'Hello.txt'}) file1.SetContentString('Hello') - file1.Upload() # Files.insert() + file1.Upload() # Files.create() - file1['title'] = 'HelloWorld.txt' # Change title of the file - file1.Upload() # Files.patch() + file1['name'] = 'HelloWorld.txt' # Change name of the file + file1.Upload() # Files.update() content = file1.GetContentString() # 'Hello' file1.SetContentString(content+' World!') # 'Hello World!' @@ -97,12 +97,12 @@ efficient way. file2 = drive.CreateFile() file2.SetContentFile('hello.png') file2.Upload() - print('Created file %s with mimeType %s' % (file2['title'], + print('Created file %s with mimeType %s' % (file2['name'], file2['mimeType'])) # Created file hello.png with mimeType image/png file3 = drive.CreateFile({'id': file2['id']}) - print('Downloading file %s from Google Drive' % file3['title']) + print('Downloading file %s from Google Drive' % file3['name']) # 'hello.png' file3.GetContentFile('world.png') # Save Drive file as a local file @@ -120,15 +120,15 @@ File listing pagination made easy # Auto-iterate through all files that matches this query file_list = drive.ListFile({'q': "'root' in parents"}).GetList() for file1 in file_list: - print('title: {}, id: {}'.format(file1['title'], file1['id'])) + print('name: {}, id: {}'.format(file1['name'], file1['id'])) # Paginate file lists by specifying number of max results - for file_list in drive.ListFile({'maxResults': 10}): + for file_list in drive.ListFile({'pageSize': 10}): print( 'Received {} files from Files.list()'.format(len(file_list)) ) # <= 10 for file1 in file_list: - print('title: {}, id: {}'.format(file1['title'], file1['id'])) + print('name: {}, id: {}'.format(file1['name'], file1['id'])) Fsspec filesystem ----------------- @@ -172,4 +172,4 @@ Thanks to all our contributors! .. _google-api-python-client: https://github.com/google/google-api-python-client -.. _Google Drive API V2: https://developers.google.com/drive/v2/web/about-sdk +.. _Google Drive API v3: https://developers.google.com/workspace/drive/api/guides/about-sdk diff --git a/docs/filelist.rst b/docs/filelist.rst index 41cd4fc..e9b7f46 100644 --- a/docs/filelist.rst +++ b/docs/filelist.rst @@ -11,12 +11,12 @@ Pass the `files.list query parameters`_ to :meth:`~pydrive2.drive.GoogleDrive.ListFile` as a dictionary. It returns a :class:`~pydrive2.files.GoogleDriveFileList`. :meth:`~pydrive2.apiattr.ApiResourceList.GetList` fetches all matches when -you omit ``maxResults``. If you set ``maxResults``, each call fetches one page. +you omit ``pageSize``. If you set ``pageSize``, each call fetches one page. -Google's `file search guide`_ explains the ``q`` parameter using Drive API -v3 examples. PyDrive2 uses v2. When adapting a query, check the -`field comparison`_ for renamed fields. For example, use ``title`` for -``name`` and ``modifiedDate`` for ``modifiedTime``. +Use Drive v3 names such as ``name`` and ``modifiedTime`` in the +`file search guide`_. Request specific fields with a field mask, for +example ``{"fields": "files(id,name,parents)"}``. PyDrive2 requests the +next page token automatically so field selection preserves pagination. .. code-block:: python @@ -30,9 +30,10 @@ v3 examples. PyDrive2 uses v2. When adapting a query, check the {'q': "'root' in parents and trashed=false"} ).GetList() for file1 in file_list: - print('title: %s, id: %s' % (file1['title'], file1['id'])) + print('name: %s, id: %s' % (file1['name'], file1['id'])) -You can update the metadata or content of each returned file. +To update a returned file, include ``id`` in any custom field mask, such as +``files(id,name)``. The default listing already includes the ID. Paginate and iterate through files ---------------------------------- @@ -44,15 +45,13 @@ contains up to ten files. .. code-block:: python # Paginate file lists by specifying number of max results - for file_list in drive.ListFile({'q': 'trashed=true', 'maxResults': 10}): + for file_list in drive.ListFile({'q': 'trashed=true', 'pageSize': 10}): print('Received %s files from Files.list()' % len(file_list)) # <= 10 for file1 in file_list: - print('title: %s, id: %s' % (file1['title'], file1['id'])) + print('name: %s, id: %s' % (file1['name'], file1['id'])) .. _files.list query parameters: - https://developers.google.com/workspace/drive/api/reference/rest/v2/files/list#query-parameters + https://developers.google.com/workspace/drive/api/reference/rest/v3/files/list#query-parameters .. _file search guide: https://developers.google.com/workspace/drive/api/guides/search-files -.. _field comparison: - https://developers.google.com/workspace/drive/api/guides/v2-to-v3-reference#resource_field_differences_between_v2_and_v3 diff --git a/docs/filemanagement.rst b/docs/filemanagement.rst index ba6d607..3186f38 100644 --- a/docs/filemanagement.rst +++ b/docs/filemanagement.rst @@ -19,11 +19,11 @@ In this example, ``gauth`` is an authenticated # Create GoogleDrive instance with authenticated GoogleAuth instance. drive = GoogleDrive(gauth) - # Create GoogleDriveFile instance with title 'Hello.txt'. - file1 = drive.CreateFile({'title': 'Hello.txt'}) + # Create GoogleDriveFile instance with name 'Hello.txt'. + file1 = drive.CreateFile({'name': 'Hello.txt'}) file1.Upload() # Upload the file. - print('title: %s, id: %s' % (file1['title'], file1['id'])) - # title: Hello.txt, id: {{FILE_ID}} + print('name: %s, id: %s' % (file1['name'], file1['id'])) + # name: Hello.txt, id: {{FILE_ID}} :meth:`~pydrive2.drive.GoogleDrive.CreateFile` creates a local file object with the drive's credentials. Call @@ -54,17 +54,17 @@ Update file metadata You can update the metadata on a :class:`~pydrive2.files.GoogleDriveFile` as you would update a ``dict``. -The `Drive API v2 file resource`_ describes the metadata fields. +The `Drive API v3 file resource`_ describes the metadata fields. Sample code continues from `Upload a new file`_: .. code-block:: python - file1['title'] = 'HelloWorld.txt' # Change title of the file. + file1['name'] = 'HelloWorld.txt' # Change name of the file. file1.Upload() # Update metadata. - print('title: %s' % file1['title']) # title: HelloWorld.txt. + print('name: %s' % file1['name']) # name: HelloWorld.txt. -Now, the title of your file has changed to 'HelloWorld.txt'. +Now, the name of your file has changed to 'HelloWorld.txt'. Download file metadata from file ID ----------------------------------- @@ -79,8 +79,8 @@ Sample code continues from above: # Create GoogleDriveFile instance with file id of file1. file2 = drive.CreateFile({'id': file1['id']}) - print('title: %s, mimeType: %s' % (file2['title'], file2['mimeType'])) - # title: HelloWorld.txt, mimeType: text/plain + print('name: %s, mimeType: %s' % (file2['name'], file2['mimeType'])) + # name: HelloWorld.txt, mimeType: text/plain Handling special metadata ------------------------- @@ -104,9 +104,9 @@ PyDrive gives you access to the metadata of an object through # Fetches the 'permissions' metadata field. file1.FetchMetadata(fields='permissions') # You can update a list of specific fields like this: - file1.FetchMetadata(fields='permissions,labels,mimeType') + file1.FetchMetadata(fields='permissions,trashed,mimeType') -See the `Drive API v2 file resource`_ for the available fields. +See the `Drive API v3 file resource`_ for the available fields. Insert permissions __________________ @@ -122,15 +122,15 @@ readable to all who have a link to the file. # Insert the permission. permission = file1.InsertPermission({ 'type': 'anyone', - 'value': 'anyone', + 'allowFileDiscovery': False, 'role': 'reader'}) - print(file1['alternateLink']) # Display the sharable link. + print(file1['webViewLink']) # Display the sharable link. Note: ``InsertPermission()`` calls ``GetPermissions()`` after successfully inserting the permission. -See the `permissions.insert request body`_ for the permission format. +See the `permissions.create request body`_ for the permission format. This file is now shared and anyone with the link can view it. But what if you want to check whether a file is already shared? @@ -180,11 +180,9 @@ delete one permission at a time by providing the permission's ID. Get files by complex queries ---------------------------- -Multiple files can have the same title. To narrow the results, combine -``title`` with another field such as ``mimeType``. Google's -`query string examples`_ use Drive API v3 field names. When adapting them -for PyDrive2, use the corresponding v2 names. For example, the filename -field in the query below is ``title``. +Multiple files can have the same name. Combine ``name`` with another +field such as ``mimeType`` in a Drive v3 search. See the +`query string examples`_ for additional filters. .. code-block:: python @@ -194,7 +192,7 @@ field in the query below is ``title``. filename = 'file_test' mimetype = 'text/plain' # Query - query = {'q': f"title = '{filename}' and mimeType='{mimetype}'"} + query = {'q': f"name = '{filename}' and mimeType='{mimetype}'"} # Get list of files that match against the query files = drive.ListFile(query).GetList() @@ -215,6 +213,40 @@ Revisions can be fetched using the ``GetRevisions()`` function of a Not all files objects have revisions. If GetRevisions is called on a file object that does not have revisions, an exception will be raised. +Upgrading from Drive API v2 +--------------------------- + +PyDrive2 uses Drive API v3 for all requests. Existing file and folder IDs +remain valid. Use v3 names in metadata dictionaries, field masks, and queries: + +- ``title`` becomes ``name``, ``fileSize`` becomes ``size``, and + ``alternateLink`` becomes ``webViewLink``. +- File reads and listings request a subset of metadata by default. To read + additional fields, use ``FetchMetadata(fields="owners,capabilities")``, + ``FetchMetadata(fetch_all=True)``, or a list selector such as + ``files(id,name,shortcutDetails)``. +- Read ``trashed`` directly and supply parent IDs as strings, + for example ``{"parents": [folder_id]}``. Assign the new parent list and + call ``Upload()`` to move an existing file. +- Replace ``maxResults`` with ``pageSize`` and use a list field mask such + as ``files(id,name)``. See :doc:`filelist` for automatic pagination. +- For user or group permissions, use ``emailAddress`` in place of + ``value``. For link sharing, set ``allowFileDiscovery=False``. + When calling ``InsertPermission(permission, param=...)``, rename the v2 + request parameter ``sendNotificationEmails`` to ``sendNotificationEmail``. + For example, ``param={"sendNotificationEmail": False}`` disables the + notification when sharing with a user or group. +- To import a file, replace the v2 ``convert`` flag with the target + Workspace MIME type and pass the original media type to ``Upload()``. + +.. code-block:: python + + sheet = drive.CreateFile({"name": "Sales", "mimeType": "application/vnd.google-apps.spreadsheet"}) + sheet.SetContentFile("sales.csv") + sheet.Upload(media_mime_type="text/csv") + +See Google's `Drive v2 to v3 reference`_ for the full field and method-parameter mappings. + Upload and update file content ------------------------------ @@ -227,7 +259,7 @@ Sample code continues from `Download file metadata from file ID`_: .. code-block:: python file4 = drive.CreateFile({ - 'title': 'appdata.json', + 'name': 'appdata.json', 'mimeType': 'application/json', }) file4.SetContentString('{"firstname": "John", "lastname": "Smith"}') @@ -239,8 +271,8 @@ Sample code continues from `Download file metadata from file ID`_: # Read file and set it as a content of this instance. file5.SetContentFile('cat.png') file5.Upload() # Upload the file. - print('title: %s, mimeType: %s' % (file5['title'], file5['mimeType'])) - # title: cat.png, mimeType: image/png + print('name: %s, mimeType: %s' % (file5['name'], file5['mimeType'])) + # name: cat.png, mimeType: image/png :meth:`~pydrive2.files.GoogleDriveFile.SetContentString` and :meth:`~pydrive2.files.GoogleDriveFile.GetContentString` accept an optional @@ -272,7 +304,7 @@ without first saving it to disk and calling drive = GoogleDrive(gauth) # Define file name and type metadata = { - 'title': 'image_test', + 'name': 'image_test', 'mimeType': 'image/jpeg' } # Create file @@ -295,10 +327,8 @@ Save the image from the previous example into a specific folder``:`` .. code-block:: python metadata = { - 'parents': [ - {"id": id_drive_folder} - ], - 'title': 'image_test', + 'parents': [id_drive_folder], + 'name': 'image_test', 'mimeType': 'image/jpeg' } # Create file @@ -310,7 +340,9 @@ Download file content Use :meth:`~pydrive2.files.GoogleDriveFile.GetContentFile` to download to disk or :meth:`~pydrive2.files.GoogleDriveFile.GetContentString` to read -the content as text. +the content as text. Disk downloads open the destination after the first +successful read. If a later read fails, the file contains the chunks +already written. Sample code continues from above: @@ -353,12 +385,15 @@ cannot be downloaded' error, set 'acknowledge_abuse=True' parameter in `GetContentFile()`. By using it you indicate that you acknowledge the risks of downloading potential malware. -.. _Drive API v2 file resource: - https://developers.google.com/workspace/drive/api/reference/rest/v2/files#resource:-file -.. _permissions.insert request body: - https://developers.google.com/workspace/drive/api/reference/rest/v2/permissions/insert#request-body +.. _Drive API v3 file resource: + https://developers.google.com/workspace/drive/api/reference/rest/v3/files#resource:-file +.. _permissions.create request body: + https://developers.google.com/workspace/drive/api/reference/rest/v3/permissions/create#request-body .. _query string examples: https://developers.google.com/workspace/drive/api/guides/search-files#examples .. _add a byte order mark to plain-text exports: https://github.com/googleapis/google-api-nodejs-client/issues/2404 .. _`abusive`: https://support.google.com/docs/answer/148505 + +.. _Drive v2 to v3 reference: + https://developers.google.com/workspace/drive/api/guides/v2-to-v3-reference diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 37b32ca..b6a15a0 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -56,13 +56,13 @@ call for you. Add the following code to your *quickstart.py* and run it. drive = GoogleDrive(gauth) - # Create GoogleDriveFile instance with title 'Hello.txt'. - file1 = drive.CreateFile({'title': 'Hello.txt'}) + # Create GoogleDriveFile instance with name 'Hello.txt'. + file1 = drive.CreateFile({'name': 'Hello.txt'}) # Set content of the file from given string. file1.SetContentString('Hello World!') file1.Upload() -This code will create a new file with title *Hello.txt* and its content *Hello +This code will create a new file with name *Hello.txt* and its content *Hello World!*. You can see and open this file from `Google Drive`_ if you want. For more details, take a look at documentation: `File management made easy`_ @@ -74,7 +74,7 @@ Listing Files ------------- *PyDrive2* handles paginations and parses response as list of -`GoogleDriveFile`_. Let's get title and id of all the files in the root folder +`GoogleDriveFile`_. Let's get name and id of all the files in the root folder of Google Drive. Again, add the following code to *quickstart.py* and execute it. @@ -85,7 +85,7 @@ it. {'q': "'root' in parents and trashed=false"} ).GetList() for file1 in file_list: - print('title: %s, id: %s' % (file1['title'], file1['id'])) + print('name: %s, id: %s' % (file1['name'], file1['id'])) Creating a Folder ----------------- @@ -98,40 +98,40 @@ The code below allows you to add a subfolder to an existing folder. def create_folder(parent_folder_id, subfolder_name): newFolder = drive.CreateFile({ - 'title': subfolder_name, - "parents": [{"kind": "drive#fileLink", "id": parent_folder_id}], + 'name': subfolder_name, + "parents": [parent_folder_id], "mimeType": "application/vnd.google-apps.folder", }) newFolder.Upload() return newFolder -Return File ID via File Title +Return File ID via File Name ----------------------------- A common task is providing the Google Drive API with a file id. -``get_id_of_title`` demonstrates a simple workflow to return the id of a file -handle by searching the file titles in a given directory. The function takes -two arguments, ``title`` and ``parent_directory_id``. ``title`` is a string -that will be compared against file titles included in a directory identified by +``get_id_of_name`` demonstrates a simple workflow to return the id of a file +handle by searching the file names in a given directory. The function takes +two arguments, ``name`` and ``parent_directory_id``. ``name`` is a string +that will be compared against file names included in a directory identified by the ``parent_directory_id``. .. code-block:: python - def get_id_of_title(title, parent_directory_id): + def get_id_of_name(name, parent_directory_id): foldered_list = drive.ListFile({ 'q': "'" + parent_directory_id + "' in parents and trashed=false" }).GetList() for file in foldered_list: - if file['title'] == title: + if file['name'] == name: return file['id'] - return None + return None Browse Folders -------------- This returns a json output of the data in a directory with some important -attributes like size, title, parent_id. +attributes like size, name, parent_id. .. code-block:: python @@ -140,7 +140,7 @@ attributes like size, title, parent_id. def folder_browser(folder_list, parent_id): for element in folder_list: if type(element) is dict: - print(element['title']) + print(element['name']) else: print(element) print( @@ -166,7 +166,7 @@ attributes like size, title, parent_id. folder_selected = inp for element in folder_list: if type(element) is dict: - if element["title"] == folder_selected: + if element["name"] == folder_selected: struc = element["list"] browsed.append(folder_selected) print("Inside " + folder_selected) @@ -174,7 +174,7 @@ attributes like size, title, parent_id. here ``folder_list`` is the list of folders that is present -You will see title and id of all the files and folders in root folder of your +You will see name and id of all the files and folders in root folder of your Google Drive. For more details, refer to the documentation: `File listing made easy`_ diff --git a/examples/Upload-and-autoconvert-to-Google-Drive-Format-Example/upload.py b/examples/Upload-and-autoconvert-to-Google-Drive-Format-Example/upload.py index 5704a12..b4aea29 100644 --- a/examples/Upload-and-autoconvert-to-Google-Drive-Format-Example/upload.py +++ b/examples/Upload-and-autoconvert-to-Google-Drive-Format-Example/upload.py @@ -8,6 +8,7 @@ example usage: upload.py 0B5XXXXY9KddXXXXXXXA2c3ZXXXX /path/to/my/file """ +import mimetypes import sys from os import path @@ -34,25 +35,30 @@ myFilePath = sys.argv[2] myFileName = path.basename(sys.argv[2]) -# Check if file name already exists in folder. +# Drive query string literals escape backslashes and single quotes. +escaped_name = myFileName.replace("\\", "\\\\").replace("'", "\\'") file_list = drive.ListFile( { - "q": f'"{parentId}" in parents and title="{myFileName}" and trashed=false' + "q": f"'{parentId}' in parents and name='{escaped_name}' and trashed=false" } ).GetList() # If file is found, update it, otherwise create new file. -if len(file_list) == 1: +if len(file_list) > 1: + sys.exit("Multiple files match. Choose a unique destination name.") +if file_list: myFile = file_list[0] else: - myFile = drive.CreateFile( - {"parents": [{"kind": "drive#fileLink", "id": parentId}]} - ) + myFile = drive.CreateFile({"parents": [parentId]}) # Upload new file content. myFile.SetContentFile(myFilePath) -myFile["title"] = myFileName -# The `convert` flag indicates to Google Drive whether to convert the -# uploaded file into a Google Drive native format, i.e. Google Sheet for -# CSV or Google Document for DOCX. -myFile.Upload({"convert": True}) +myFile["name"] = myFileName +source_type = mimetypes.guess_type(myFilePath)[0] or "application/octet-stream" +if not file_list: + targets = drive.GetAbout(fields="importFormats")["importFormats"].get( + source_type, [] + ) + if targets: + myFile["mimeType"] = targets[0] +myFile.Upload(media_mime_type=source_type) diff --git a/examples/strip_bom_example.py b/examples/strip_bom_example.py index b595bd6..e1a1ae8 100644 --- a/examples/strip_bom_example.py +++ b/examples/strip_bom_example.py @@ -10,8 +10,8 @@ file1 = drive.CreateFile() original_file_content = "Generic, non-exhaustive\n ASCII test string." file1.SetContentString(original_file_content) -# {'convert': True} triggers conversion to a Google Drive document. -file1.Upload({"convert": True}) +file1["mimeType"] = "application/vnd.google-apps.document" +file1.Upload(media_mime_type="text/plain") # Download the file. diff --git a/examples/using_folders.py b/examples/using_folders.py index 58448ed..19f7876 100644 --- a/examples/using_folders.py +++ b/examples/using_folders.py @@ -8,7 +8,7 @@ # Create folder. folder_metadata = { - "title": "", + "name": "", # The mimetype defines this new file as a folder, so don't change this. "mimeType": "application/vnd.google-apps.folder", } @@ -16,14 +16,12 @@ folder.Upload() # Get folder info and print to screen. -folder_title = folder["title"] +folder_name = folder["name"] folder_id = folder["id"] -print(f"title: {folder_title}, id: {folder_id}") +print(f"name: {folder_name}, id: {folder_id}") # Upload file to folder. -f = drive.CreateFile( - {"parents": [{"kind": "drive#fileLink", "id": folder_id}]} -) +f = drive.CreateFile({"parents": [folder_id]}) # Make sure to add the path to the file to upload below. f.SetContentFile("") diff --git a/pydrive2/__pyinstaller/test_hook-googleapiclient.py b/pydrive2/__pyinstaller/test_hook-googleapiclient.py index 4fca8fa..b7b4c55 100644 --- a/pydrive2/__pyinstaller/test_hook-googleapiclient.py +++ b/pydrive2/__pyinstaller/test_hook-googleapiclient.py @@ -11,7 +11,7 @@ documents = files("googleapiclient.discovery_cache.documents") -assert documents.joinpath("drive.v2.json").is_file() +assert documents.joinpath("drive.v3.json").is_file() """ diff --git a/pydrive2/apiattr.py b/pydrive2/apiattr.py index d025434..b022df6 100644 --- a/pydrive2/apiattr.py +++ b/pydrive2/apiattr.py @@ -78,14 +78,6 @@ def __repr__(self): dict_representation = dict.__repr__(self) return f"{type(self).__name__}({dict_representation})" - def update(self, *args, **kwargs): - """Overwritten method of dictionary.""" - BAD_URL_PREFIX = "https://www.googleapis.comhttps:" - for k, v in dict(*args, **kwargs).items(): - if k == "downloadUrl" and v.startswith(BAD_URL_PREFIX): - v = v.replace(BAD_URL_PREFIX, "https://www.googleapis.com", 1) - self[k] = v - def UpdateMetadata(self, metadata=None): """Update metadata and mark all of them to be clean.""" if metadata: @@ -155,19 +147,18 @@ def __next__(self): def GetList(self): """Get list of API resources. - If 'maxResults' is not specified, it will automatically iterate through + If 'pageSize' is not specified, it will automatically iterate through every resources available. Otherwise, it will make API call once and update 'pageToken'. :returns: list -- list of API resources. """ - if self.get("maxResults") is None: - self["maxResults"] = 1000 - result = [] - for x in self: - result.extend(x) - del self["maxResults"] - return result + if self.get("pageSize") is None: + self["pageSize"] = 1000 + try: + return [item for page in self for item in page] + finally: + del self["pageSize"] else: return next(self) diff --git a/pydrive2/auth.py b/pydrive2/auth.py index d53a2f1..f2685d0 100644 --- a/pydrive2/auth.py +++ b/pydrive2/auth.py @@ -5,6 +5,7 @@ import webbrowser from contextlib import suppress from functools import wraps +from inspect import signature from pathlib import Path from typing import ClassVar @@ -52,6 +53,8 @@ class RefreshError(AuthError): def LoadAuth(decoratee): """Decorator to check if the auth is valid and loads auth if not.""" + call_signature = signature(decoratee) + @wraps(decoratee) def _decorated(self, *args, **kwargs): # Initialize auth if needed. @@ -76,16 +79,17 @@ def _decorated(self, *args, **kwargs): self.auth.Authorize() # Ensure that a thread-safe HTTP object is provided. - if ( - kwargs is not None - and "param" in kwargs - and kwargs["param"] is not None - and "http" in kwargs["param"] - and kwargs["param"]["http"] is not None - ): - self.http = kwargs["param"]["http"] - del kwargs["param"]["http"] - + transport = None + if "param" in call_signature.parameters: + call = call_signature.bind(self, *args, **kwargs) + params = call.arguments.get("param") + if params and "http" in params: + params = dict(params) + transport = params.pop("http") + call.arguments["param"] = params + args, kwargs = call.args[1:], call.kwargs + if transport is not None: + self.http = transport else: # If HTTP object not specified, create or resuse an HTTP # object from the thread local storage. @@ -681,7 +685,7 @@ def Authorize(self): self.http = self.http.http self.http = AuthorizedHttp(self.credentials, http=self.http) self.service = build( - "drive", "v2", http=self.http, cache_discovery=False + "drive", "v3", http=self.http, cache_discovery=False ) def Get_Http_Object(self): diff --git a/pydrive2/drive.py b/pydrive2/drive.py index 9a27224..b8a6f84 100644 --- a/pydrive2/drive.py +++ b/pydrive2/drive.py @@ -40,9 +40,14 @@ def ListFile(self, param=None): return GoogleDriveFileList(auth=self.auth, param=param) @LoadAuth - def GetAbout(self): + def GetAbout(self, fields="*"): """Return information about the Google Drive of the auth instance. + :param fields: Drive About fields to return. Defaults to all fields. :returns: A dictionary of Google Drive information like user, usage, quota etc. """ - return self.auth.service.about().get().execute(http=self.http) + return ( + self.auth.service.about() + .get(fields=fields or "*") + .execute(http=self.http) + ) diff --git a/pydrive2/files.py b/pydrive2/files.py index 0dddceb..acad9c9 100644 --- a/pydrive2/files.py +++ b/pydrive2/files.py @@ -1,3 +1,4 @@ +import codecs import io import json import mimetypes @@ -19,13 +20,42 @@ ) from .auth import LoadAuth -BLOCK_SIZE = 1024 +DEFAULT_FILE_FIELDS = ( + "id,name,mimeType,description,parents,trashed,size,md5Checksum," + "resourceKey,webViewLink,createdTime,modifiedTime" +) + +WORKSPACE_EXPORT_TYPES = frozenset( + { + "application/vnd.google-apps.document", + "application/vnd.google-apps.spreadsheet", + "application/vnd.google-apps.presentation", + "application/vnd.google-apps.drawing", + } +) + # Usage: MIME_TYPE_TO_BOM['']['']. MIME_TYPE_TO_BOM = { "application/vnd.google-apps.document": {"text/plain": "\ufeff".encode()} } +def _ensure_field(fields, required): + """Include a top-level field in a Drive partial-response selector.""" + fields = fields.strip() + start = depth = 0 + for index, char in enumerate(fields + ","): + if char == "(": + depth += 1 + elif char == ")": + depth -= 1 + elif char == "," and depth == 0: + if fields[start:index].strip() in (required, "*"): + return fields + start = index + 1 + return f"{required},{fields}" if fields else required + + class FileNotUploadedError(RuntimeError): """Error trying to access metadata of file that is not uploaded.""" @@ -33,15 +63,21 @@ class FileNotUploadedError(RuntimeError): class ApiRequestError(IOError): def __init__(self, http_error): assert isinstance(http_error, errors.HttpError) - content = json.loads(http_error.content.decode("utf-8")) - self.error = content.get("error", {}) if content else {} + try: + content = json.loads(http_error.content) + except (ValueError, UnicodeError): + content = {} + error = content.get("error") if isinstance(content, dict) else None + self.error = dict(error) if isinstance(error, dict) else {} + self.error.setdefault("code", http_error.resp.status) # Initialize args for backward compatibility super().__init__(http_error) def GetField(self, field): """Returns the `field` from the first error""" - return self.error.get("errors", [{}])[0].get(field, "") + details = self.error.get("errors") or [{}] + return details[0].get(field, "") class FileNotDownloadableError(RuntimeError): @@ -79,44 +115,39 @@ def _GetList(self): :returns: list -- list of pydrive2.files.GoogleDriveFile. """ - # Teamdrive support - self["supportsAllDrives"] = True - self["includeItemsFromAllDrives"] = True - + params = dict( + self, supportsAllDrives=True, includeItemsFromAllDrives=True + ) + fields = params.get("fields") or f"files({DEFAULT_FILE_FIELDS})" + # Keep the cursor even when the caller selects only file fields. + params["fields"] = _ensure_field(fields, "nextPageToken") try: self.metadata = ( self.auth.service.files() - .list(**dict(self)) + .list(**params) .execute(http=self.http) ) except errors.HttpError as error: raise ApiRequestError(error) - result = [] - for file_metadata in self.metadata["items"]: - tmp_file = GoogleDriveFile( - auth=self.auth, metadata=file_metadata, uploaded=True - ) - result.append(tmp_file) - return result + return [ + GoogleDriveFile(self.auth, metadata, uploaded=True) + for metadata in self.metadata.get("files", []) + ] class IoBuffer: """Lightweight retention of one chunk.""" - def __init__(self, encoding): - self.encoding = encoding - self.chunk = None + def __init__(self): + self.chunk = b"" def write(self, chunk): self.chunk = chunk def read(self): - return ( - self.chunk.decode(self.encoding) - if self.chunk and self.encoding - else self.chunk - ) + chunk, self.chunk = self.chunk, b"" + return chunk class MediaIoReadable: @@ -136,46 +167,53 @@ def __init__( :raises: ApiRequestError """ self.done = False - self._fd = IoBuffer(encoding) + self._fd = IoBuffer() self.downloader = MediaIoBaseDownload( self._fd, request, chunksize=chunksize ) self.size = None - self._pre_buffer = False - if pre_buffer: - self.read() - if remove_prefix: - chunk = io.BytesIO(self._fd.chunk or b"") - GoogleDriveFile._RemovePrefix(chunk, remove_prefix) - self._fd.chunk = chunk.getvalue() - self._pre_buffer = True + self.progress = 0 + self._decoder = ( + codecs.getincrementaldecoder(encoding)() if encoding else None + ) + self._prefix = remove_prefix + self._buffer = self._read_chunk() if pre_buffer else None - def read(self): - """ - :returns: bytes or str -- chunk (or None if done) - :raises: ApiRequestError - """ - if self._pre_buffer: - self._pre_buffer = False - return self._fd.read() - if self.done: - return None + def _next_chunk(self): try: status, self.done = self.downloader.next_chunk() - self.size = status.total_size except errors.HttpError as error: - raise ApiRequestError(error) + raise ApiRequestError(error) from error + self.size = status.total_size + self.progress = status.resumable_progress return self._fd.read() + def _read_chunk(self): + chunk = self._next_chunk() + if self._prefix: + while ( + not self.done + and len(chunk) < len(self._prefix) + and self._prefix.startswith(chunk) + ): + chunk += self._next_chunk() + chunk = chunk.removeprefix(self._prefix) + self._prefix = b"" + if self._decoder: + return self._decoder.decode(chunk, final=self.done) + return chunk + + def read(self): + """Return the next chunk, or None at end of stream.""" + if self._buffer is not None: + chunk, self._buffer = self._buffer, None + return chunk + if self.done: + return None + return self._read_chunk() + def __iter__(self): - """ - :raises: ApiRequestError - """ - while True: - chunk = self.read() - if chunk is None: - break - yield chunk + return iter(self.read, None) def __len__(self): return self.size @@ -213,6 +251,7 @@ def __init__(self, auth=None, metadata=None, uploaded=False): elif metadata: self.update(metadata) self.has_bom = True + self._export_mimetype = None def __getitem__(self, key): """Overwrites manner of accessing Files resource. @@ -248,6 +287,8 @@ def SetContentString(self, content, encoding="utf-8"): :param content: content of the file in string. :type content: str """ + self._export_mimetype = None + self.has_bom = True self.content = io.BytesIO(content.encode(encoding)) if self.get("mimeType") is None and self.get("id") is None: self["mimeType"] = "text/plain" @@ -257,7 +298,7 @@ def SetContentFile(self, filename): Opens the file specified by this method. Will be read, uploaded, and closed by Upload() method. - Sets metadata 'title' and 'mimeType' automatically if not specified and + Sets metadata 'name' and 'mimeType' automatically if not specified and the file is uploaded for the first time (id is not set). :param filename: name of the file to be uploaded. @@ -265,10 +306,13 @@ def SetContentFile(self, filename): """ self.content = open(filename, "rb") # noqa: SIM115 - retained for upload if self.get("id") is None: - if self.get("title") is None: - self["title"] = os.path.basename(filename) + if self.get("name") is None: + self["name"] = os.path.basename(filename) if self.get("mimeType") is None: - self["mimeType"] = mimetypes.guess_type(filename)[0] + self["mimeType"] = ( + mimetypes.guess_type(filename)[0] + or "application/octet-stream" + ) def GetContentString( self, mimetype=None, encoding="utf-8", remove_bom=False @@ -287,15 +331,13 @@ def GetContentString( :returns: str -- utf-8 decoded content of the file :raises: ApiRequestError, FileNotUploadedError, FileNotDownloadableError """ - if ( - self.content is None - or not isinstance(self.content, io.BytesIO) - or self.has_bom == remove_bom - ): + stale = not self.dirty["content"] and ( + self.has_bom == remove_bom or self._export_mimetype != mimetype + ) + if not isinstance(self.content, io.BytesIO) or stale: self.FetchContent(mimetype, remove_bom) return self.content.getvalue().decode(encoding) - @LoadAuth def GetContentFile( self, filename, @@ -307,6 +349,9 @@ def GetContentFile( ): """Save content of this file as a local file. + The destination is opened after the first successful read. If a later + read fails, the destination contains the chunks written so far. + :param filename: name of the file to write to. :type filename: str :param mimetype: mimeType of the file. @@ -322,55 +367,20 @@ def GetContentFile( :type acknowledge_abuse: bool :raises: ApiRequestError, FileNotUploadedError """ - files = self.auth.service.files() - file_id = self.metadata.get("id") or self.get("id") - if not file_id: - raise FileNotUploadedError() - - def download(fd, request): - downloader = MediaIoBaseDownload( - fd, self._WrapRequest(request), chunksize=chunksize - ) - done = False - while done is False: - status, done = downloader.next_chunk() + stream = self.GetContentIOBuffer( + mimetype=mimetype, + remove_bom=remove_bom, + chunksize=chunksize, + acknowledge_abuse=acknowledge_abuse, + ) + # The pre-buffered first response determines whether the file is readable. + with open(filename, "wb") as fd: + for chunk in stream: + fd.write(chunk) + # Release the response body before the next HTTP request. + del chunk if callback: - callback(status.resumable_progress, status.total_size) - - with open(filename, mode="w+b") as fd: - # Should use files.export_media instead of files.get_media if - # metadata["mimeType"].startswith("application/vnd.google-apps."). - # But that would first require a slow call to FetchMetadata(). - # We prefer to try-except for speed. - try: - download( - fd, - files.get_media( - fileId=file_id, acknowledgeAbuse=acknowledge_abuse - ), - ) - except errors.HttpError as error: - exc = ApiRequestError(error) - if ( - exc.error["code"] != 403 - or exc.GetField("reason") != "fileNotDownloadable" - ): - raise exc - mimetype = mimetype or "text/plain" - fd.seek(0) # just in case `download()` modified `fd` - try: - download( - fd, - files.export_media(fileId=file_id, mimeType=mimetype), - ) - except errors.HttpError as error: - raise ApiRequestError(error) - - if mimetype == "text/plain" and remove_bom: - fd.seek(0) - bom = self._GetBOM(mimetype) - if bom: - self._RemovePrefix(fd, bom) + callback(stream.progress, stream.size) @LoadAuth def GetContentIOBuffer( @@ -397,52 +407,44 @@ def GetContentIOBuffer( :returns: MediaIoReadable -- file-like object. :raises: ApiRequestError, FileNotUploadedError """ - files = self.auth.service.files() - file_id = self.metadata.get("id") or self.get("id") - if not file_id: - raise FileNotUploadedError() - # Should use files.export_media instead of files.get_media if - # metadata["mimeType"].startswith("application/vnd.google-apps."). - # But that would first require a slow call to FetchMetadata(). - # We prefer to try-except for speed. - try: - request = self._WrapRequest( - files.get_media( - fileId=file_id, acknowledgeAbuse=acknowledge_abuse - ) - ) - return MediaIoReadable( - request, encoding=encoding, chunksize=chunksize - ) - except ApiRequestError as exc: - if ( - exc.error["code"] != 403 - or exc.GetField("reason") != "fileNotDownloadable" - ): - raise - mimetype = mimetype or "text/plain" - request = self._WrapRequest( - files.export_media(fileId=file_id, mimeType=mimetype) - ) - remove_prefix = ( - self._GetBOM(mimetype) - if mimetype == "text/plain" and remove_bom - else b"" - ) + def open_stream(request, content_type): + prefix = self._GetBOM(content_type) if remove_bom else b"" return MediaIoReadable( - request, + self._WrapRequest(request), encoding=encoding, - remove_prefix=remove_prefix, + remove_prefix=prefix, chunksize=chunksize, ) + file_id = self._FileId() + files = self.auth.service.files() + source_type = self._SourceMimeType() + if source_type not in WORKSPACE_EXPORT_TYPES: + request = files.get_media( + fileId=file_id, + acknowledgeAbuse=acknowledge_abuse, + supportsAllDrives=True, + ) + try: + return open_stream(request, mimetype or source_type) + except ApiRequestError as error: + if ( + error.error.get("code") != 403 + or error.GetField("reason") != "fileNotDownloadable" + ): + raise + + export_type = mimetype or "text/plain" + request = files.export_media(fileId=file_id, mimeType=export_type) + return open_stream(request, export_type) + @LoadAuth def FetchMetadata(self, fields=None, fetch_all=False): """Download file's metadata from id using Files.get(). :param fields: The fields to include, as one string, each entry separated - by commas, e.g. 'fields,labels'. + by commas, e.g. 'id,name,trashed'. :type fields: str :param fetch_all: Whether to fetch all fields. @@ -450,79 +452,122 @@ def FetchMetadata(self, fields=None, fetch_all=False): :raises: ApiRequestError, FileNotUploadedError """ - file_id = self.metadata.get("id") or self.get("id") + file_id = self._FileId() if fetch_all: fields = "*" - if file_id: - try: - request = self.auth.service.files().get( - fileId=file_id, - fields=fields, - # Teamdrive support - supportsAllDrives=True, - ) - request = self._AddResourceKeyHeaders(request) - metadata = request.execute(http=self.http) - except errors.HttpError as error: - raise ApiRequestError(error) - else: - self.uploaded = True - self.UpdateMetadata(metadata) - else: - raise FileNotUploadedError() + request = self.auth.service.files().get( + fileId=file_id, + fields=fields or DEFAULT_FILE_FIELDS, + supportsAllDrives=True, + ) + metadata = self._Execute(request) + self.uploaded = True + self.UpdateMetadata(metadata) - @LoadMetadata def FetchContent(self, mimetype=None, remove_bom=False): - """Download file's content from download_url. - - :raises: ApiRequestError, FileNotUploadedError, FileNotDownloadableError - """ - download_url = self.metadata.get("downloadUrl") - export_links = self.metadata.get("exportLinks") - if download_url: - self.content = io.BytesIO(self._DownloadFromUrl(download_url)) - self.dirty["content"] = False - - elif export_links and export_links.get(mimetype): - self.content = io.BytesIO( - self._DownloadFromUrl(export_links.get(mimetype)) - ) - self.dirty["content"] = False - - else: + """Download file content or a Workspace export into memory.""" + if not self.uploaded and not self._SourceMimeType(): + self.FetchMetadata() + if self._SourceMimeType() in { + "application/vnd.google-apps.folder", + "application/vnd.google-apps.shortcut", + }: raise FileNotDownloadableError( - "No downloadLink/exportLinks for mimetype found in metadata" + "Folders and shortcuts have no content" ) - - if mimetype == "text/plain" and remove_bom: - self._RemovePrefix( - self.content, MIME_TYPE_TO_BOM[self["mimeType"]][mimetype] + try: + stream = self.GetContentIOBuffer( + mimetype=mimetype, remove_bom=remove_bom ) - self.has_bom = not remove_bom + self.content = io.BytesIO(b"".join(stream)) + except ApiRequestError as error: + if error.GetField("reason") in { + "fileNotExportable", + "exportFormatNotFound", + "conversionUnsupported", + "cannotExportFile", + }: + raise FileNotDownloadableError(str(error)) from error + raise + self.dirty["content"] = False + self.has_bom = not remove_bom + self._export_mimetype = mimetype - def Upload(self, param=None): - """Upload/update file by choosing the most efficient method. - - :param param: additional parameter to upload file. - :type param: dict. - :raises: ApiRequestError - """ - if self.uploaded or self.get("id") is not None: - if self.dirty["content"]: - self._FilesUpdate(param=param) - else: - self._FilesPatch(param=param) + @LoadAuth + def Upload(self, param=None, *, media_mime_type=None): + """Create or update a file, optionally importing uploaded media. + + For a Workspace import, set the destination MIME type in the file + metadata and pass the source MIME type as media_mime_type. + """ + params = dict(param or {}, supportsAllDrives=True) + params["fields"] = params.get("fields") or DEFAULT_FILE_FIELDS + body = self.GetChanges() + if len(body.get("parents", [])) > 1: + raise ValueError("A Drive v3 file can have only one parent") + file_id = self._FileId() if self.uploaded else self.get("id") + files = self.auth.service.files() + if self.dirty["content"]: + source_type = ( + media_mime_type + or self.get("mimeType") + or "application/octet-stream" + ) + params["media_body"] = MediaIoBaseUpload( + self.content, source_type, resumable=True + ) + # The destination type determines whether we can reuse the source bytes. + if not self.get("mimeType"): + params["fields"] = _ensure_field(params["fields"], "mimeType") + if file_id: + body.pop("id", None) + if "parents" in body: + self._MoveParents(body.pop("parents"), params) + request = files.update(fileId=file_id, body=body, **params) else: - self._FilesInsert(param=param) + params["fields"] = _ensure_field(params["fields"], "id") + request = files.create(body=body, **params) + metadata = self._Execute(request) + if ( + "addParents" in params or "removeParents" in params + ) and "parents" not in metadata: + self.pop("parents", None) + destination_type = ( + metadata.get("mimeType") or self.get("mimeType") or "" + ) + if self.dirty["content"] and destination_type.startswith( + "application/vnd.google-apps." + ): + self.content = None + self.uploaded = True + self.dirty["content"] = False + self.UpdateMetadata(metadata) + + def _MoveParents(self, parents, params): + """Translate a changed parent list into Drive v3 update parameters.""" + previous = self.metadata.get("parents") + if previous is None: + request = self.auth.service.files().get( + fileId=self._FileId(), fields="parents", supportsAllDrives=True + ) + previous = self._Execute(request).get("parents", []) + if previous and parents == ["root"] and "root" not in previous: + root = self.auth.service.files().get(fileId="root", fields="id") + parents = [self._Execute(root)["id"]] + if parents and parents[0] not in previous: + params["addParents"] = parents[0] + removed = [parent for parent in previous if parent not in parents] + if removed: + params["removeParents"] = ",".join(removed) def Trash(self, param=None): """Move a file to the trash. :raises: ApiRequestError """ - self._FilesTrash(param=param) + self._SetTrashed(True, param) def UnTrash(self, param=None): """Move a file out of the trash. @@ -530,8 +575,9 @@ def UnTrash(self, param=None): :type param: dict. :raises: ApiRequestError """ - self._FilesUnTrash(param=param) + self._SetTrashed(False, param) + @LoadAuth def Delete(self, param=None): """Hard-delete a file. @@ -539,7 +585,10 @@ def Delete(self, param=None): :type param: dict. :raises: ApiRequestError """ - self._FilesDelete(param=param) + params = dict( + param or {}, fileId=self._FileId(), supportsAllDrives=True + ) + self._Execute(self.auth.service.files().delete(**params)) @LoadAuth def Copy(self, target_folder=None, new_title=None, param=None): @@ -556,32 +605,27 @@ def Copy(self, target_folder=None, new_title=None, param=None): :rtype: GoogleDriveFile """ - if param is None: - param = {} - - param["fileId"] = self["id"] - param["supportsAllDrives"] = True - param["body"] = {} - - if target_folder: - param["body"]["parents"] = [{"id": target_folder["id"]}] - param["body"]["title"] = new_title - - new_file = None - try: - new_file = ( - self.auth.service.files().copy(**param).execute(http=self.http) - ) - except errors.HttpError as error: - raise ApiRequestError(error) - - return GoogleDriveFile(self.auth, new_file) + params = dict(param or {}, supportsAllDrives=True) + params["fields"] = _ensure_field( + params.get("fields") or DEFAULT_FILE_FIELDS, "id" + ) + body = {} + if target_folder is not None: + body["parents"] = [target_folder["id"]] + if new_title is not None: + body["name"] = new_title + request = self.auth.service.files().copy( + fileId=self._FileId(), body=body, **params + ) + metadata = self._Execute(request) + return GoogleDriveFile(self.auth, metadata, uploaded=True) + @LoadAuth def InsertPermission(self, new_permission, param=None): """Insert a new permission. Re-fetches all permissions after call. :param new_permission: The new permission to insert, please see the - official Google Drive API guide on permissions.insert + official Google Drive API guide on permissions.create for details. :type new_permission: object @@ -591,57 +635,32 @@ def InsertPermission(self, new_permission, param=None): :return: The permission object. :rtype: object """ - if param is None: - param = {} - param["fileId"] = self.metadata.get("id") or self["id"] - param["body"] = new_permission - # Teamdrive support - param["supportsAllDrives"] = True - - try: - permission = ( - self.auth.service.permissions() - .insert(**param) - .execute(http=self.http) - ) - except errors.HttpError as error: - raise ApiRequestError(error) - else: - self.GetPermissions() # Update permissions field. - + params = dict( + param or {}, + fileId=self._FileId(), + body=new_permission, + supportsAllDrives=True, + ) + params["fields"] = params.get("fields") or "*" + permission = self._Execute( + self.auth.service.permissions().create(**params) + ) + self._ListPermissions() return permission @LoadAuth def GetPermissions(self): - """Get file's or shared drive's permissions. - - For files in a shared drive, at most 100 results will be returned. - It doesn't paginate and collect all results. - - :return: A list of the permission objects. - :rtype: object[] - """ - file_id = self.metadata.get("id") or self.get("id") - - # We can't do FetchMetada call (which would nicely update - # local metada cache, etc) here since it doesn't return - # permissions for the team drive use case. - permissions = ( - self.auth.service.permissions() - .list( - fileId=file_id, - # Teamdrive support - supportsAllDrives=True, - ) - .execute(http=self.http) - ).get("items") - - if permissions: - self["permissions"] = permissions - self.metadata["permissions"] = permissions - - return permissions + """Return all permissions for the file, including shared drive files.""" + return self._ListPermissions() + + def _ListPermissions(self): + return self._ListChildren( + self.auth.service.permissions(), + "permissions", + supportsAllDrives=True, + ) + @LoadAuth def DeletePermission(self, permission_id): """Deletes the permission specified by the permission_id. @@ -650,36 +669,55 @@ def DeletePermission(self, permission_id): :return: True if it succeeds. :rtype: bool """ - return self._DeletePermission(permission_id) + request = self.auth.service.permissions().delete( + fileId=self._FileId(), + permissionId=permission_id, + supportsAllDrives=True, + ) + self._Execute(request) + if "permissions" in self and "permissions" in self.metadata: + permissions = [ + permission + for permission in self["permissions"] + if permission["id"] != permission_id + ] + self["permissions"] = self.metadata["permissions"] = permissions + return True @LoadAuth def GetRevisions(self): - """Get file's or shared drive's revisions. + """Return all revisions visible to the authenticated user.""" + return self._ListChildren(self.auth.service.revisions(), "revisions") + + def _ListChildren(self, resource, key, **params): + params.update(fileId=self._FileId(), fields=f"nextPageToken,{key}(*)") + items = [] + while True: + request = resource.list(**params) + page = self._Execute(request) + items.extend(page.get(key, [])) + if not page.get("nextPageToken"): + break + params["pageToken"] = page["nextPageToken"] + self[key] = self.metadata[key] = items + return items - For files in a shared drive, at most 100 results will be returned. - It doesn't paginate and collect all results. + def _SourceMimeType(self): + return self.metadata.get("mimeType") or self.get("mimeType") - :return: A list of the revision objects. - :rtype: object[] - """ + def _FileId(self): file_id = self.metadata.get("id") or self.get("id") - - # We can't do FetchMetada call (which would nicely update - # local metada cache, etc) here since it doesn't return - # revisions for the team drive use case. - revisions = ( - self.auth.service.revisions() - .list( - fileId=file_id, + if not file_id: + raise FileNotUploadedError( + "This operation requires a Drive file ID." ) - .execute(http=self.http) - ).get("items") - - if revisions: - self["revisions"] = revisions - self.metadata["revisions"] = revisions + return file_id - return revisions + def _Execute(self, request): + try: + return self._AddResourceKeyHeaders(request).execute(http=self.http) + except errors.HttpError as error: + raise ApiRequestError(error) from error def _WrapRequest(self, request): """Replaces request.http with self.http. @@ -709,218 +747,14 @@ def _AddResourceKeyHeaders(self, request): return request @LoadAuth - def _FilesInsert(self, param=None): - """Upload a new file using Files.insert(). - - :param param: additional parameter to upload file. - :type param: dict. - :raises: ApiRequestError - """ - if param is None: - param = {} - param["body"] = self.GetChanges() - - # teamdrive support - param["supportsAllDrives"] = True - - try: - if self.dirty["content"]: - param["media_body"] = self._BuildMediaBody() - metadata = ( - self.auth.service.files() - .insert(**param) - .execute(http=self.http) - ) - except errors.HttpError as error: - raise ApiRequestError(error) - else: - self.uploaded = True - self.dirty["content"] = False - self.UpdateMetadata(metadata) - - @LoadAuth - def _FilesUnTrash(self, param=None): - """Un-delete (Trash) a file using Files.UnTrash(). - :param param: additional parameter to file. - :type param: dict. - :raises: ApiRequestError - """ - if param is None: - param = {} - param["fileId"] = self.metadata.get("id") or self["id"] - - # Teamdrive support - param["supportsAllDrives"] = True - - try: - self.auth.service.files().untrash(**param).execute(http=self.http) - except errors.HttpError as error: - raise ApiRequestError(error) - else: - if self.metadata: - self.metadata["labels"]["trashed"] = False - return True - - @LoadAuth - def _FilesTrash(self, param=None): - """Soft-delete (Trash) a file using Files.Trash(). - - :param param: additional parameter to file. - :type param: dict. - :raises: ApiRequestError - """ - if param is None: - param = {} - param["fileId"] = self.metadata.get("id") or self["id"] - - # Teamdrive support - param["supportsAllDrives"] = True - - try: - self.auth.service.files().trash(**param).execute(http=self.http) - except errors.HttpError as error: - raise ApiRequestError(error) - else: - if self.metadata: - self.metadata["labels"]["trashed"] = True - return True - - @LoadAuth - def _FilesDelete(self, param=None): - """Delete a file using Files.Delete() - (WARNING: deleting permanently deletes the file!) - - :param param: additional parameter to file. - :type param: dict. - :raises: ApiRequestError - """ - if param is None: - param = {} - param["fileId"] = self.metadata.get("id") or self["id"] - - # Teamdrive support - param["supportsAllDrives"] = True - - try: - self.auth.service.files().delete(**param).execute(http=self.http) - except errors.HttpError as error: - raise ApiRequestError(error) - else: - return True - - @LoadAuth - def _FilesUpdate(self, param=None): - """Update metadata and/or content using Files.Update(). - - :param param: additional parameter to upload file. - :type param: dict. - :raises: ApiRequestError, FileNotUploadedError - """ - if param is None: - param = {} - param["body"] = self.GetChanges() - param["fileId"] = self.metadata.get("id") or self.get("id") - - # Teamdrive support - param["supportsAllDrives"] = True - - try: - if self.dirty["content"]: - param["media_body"] = self._BuildMediaBody() - metadata = ( - self.auth.service.files() - .update(**param) - .execute(http=self.http) - ) - except errors.HttpError as error: - raise ApiRequestError(error) - else: - self.uploaded = True - self.dirty["content"] = False - self.UpdateMetadata(metadata) - - @LoadAuth - def _FilesPatch(self, param=None): - """Update metadata using Files.Patch(). - - :param param: additional parameter to upload file. - :type param: dict. - :raises: ApiRequestError, FileNotUploadedError - """ - if param is None: - param = {} - param["body"] = self.GetChanges() - param["fileId"] = self.metadata.get("id") or self.get("id") - - # Teamdrive support - param["supportsAllDrives"] = True - - try: - metadata = ( - self.auth.service.files() - .patch(**param) - .execute(http=self.http) - ) - except errors.HttpError as error: - raise ApiRequestError(error) - else: - self.UpdateMetadata(metadata) - - def _BuildMediaBody(self): - """Build MediaIoBaseUpload to get prepared to upload content of the file. - - Sets mimeType as 'application/octet-stream' if not specified. - - :returns: MediaIoBaseUpload -- instance that will be used to upload content. - """ - if self.get("mimeType") is None: - self["mimeType"] = "application/octet-stream" - return MediaIoBaseUpload( - self.content, self["mimeType"], resumable=True + def _SetTrashed(self, trashed, param): + params = dict(param or {}, supportsAllDrives=True) + params.setdefault("fields", "trashed") + request = self.auth.service.files().update( + fileId=self._FileId(), body={"trashed": trashed}, **params ) - - @LoadAuth - def _DownloadFromUrl(self, url): - """Download file from url using provided credential. - - :param url: link of the file to download. - :type url: str. - :returns: str -- content of downloaded file in string. - :raises: ApiRequestError - """ - assert self.http is not None # Set by LoadAuth. - resp, content = self.http.request(url) - if resp.status != 200: - raise ApiRequestError(errors.HttpError(resp, content, uri=url)) - return content - - @LoadAuth - def _DeletePermission(self, permission_id): - """Deletes the permission remotely, and from the file object itself. - - :param permission_id: The ID of the permission. - :type permission_id: str - - :return: The permission - :rtype: object - """ - file_id = self.metadata.get("id") or self["id"] - try: - self.auth.service.permissions().delete( - fileId=file_id, permissionId=permission_id - ).execute(http=self.http) - except errors.HttpError as error: - raise ApiRequestError(error) - else: - if "permissions" in self and "permissions" in self.metadata: - permissions = [ - permission - for permission in self["permissions"] - if permission["id"] == permission_id - ] - self["permissions"] = permissions - self.metadata["permissions"] = permissions - return True + self._Execute(request) + self["trashed"] = self.metadata["trashed"] = trashed @staticmethod def _GetBOM(mimetype): @@ -928,81 +762,3 @@ def _GetBOM(mimetype): for bom in MIME_TYPE_TO_BOM.values(): if mimetype in bom: return bom[mimetype] - - @staticmethod - def _RemovePrefix(file_object, prefix, block_size=BLOCK_SIZE): - """Deletes passed prefix by shifting content of passed file object by to - the left. Operation is in-place. - - Args: - file_object (obj): The file object to manipulate. - prefix (str): The prefix to insert. - block_size (int): The size of the blocks which are moved one at a time. - """ - prefix_length = len(prefix) - # Detect if prefix exists in file. - content_start = file_object.read(prefix_length) - - if content_start == prefix: - # Shift content left by prefix length, by copying 1KiB at a time. - block_to_write = file_object.read(block_size) - current_block_length = len(block_to_write) - - # Read and write location in separate variables for simplicity. - read_location = prefix_length + current_block_length - write_location = 0 - - while current_block_length > 0: - # Write next block. - file_object.seek(write_location) - file_object.write(block_to_write) - # Set write location to the next block. - write_location += len(block_to_write) - - # Read next block of input. - file_object.seek(read_location) - block_to_write = file_object.read(block_size) - # Update the current block length and read_location. - current_block_length = len(block_to_write) - read_location += current_block_length - - # Truncate the file to its, now shorter, length. - file_object.truncate(read_location - prefix_length) - - @staticmethod - def _InsertPrefix(file_object, prefix, block_size=BLOCK_SIZE): - """Inserts the passed prefix in the beginning of the file, operation is - in-place. - - Args: - file_object (obj): The file object to manipulate. - prefix (str): The prefix to insert. - """ - # Read the first two blocks. - first_block = file_object.read(block_size) - second_block = file_object.read(block_size) - # Pointer to the first byte of the next block to be read. - read_location = block_size * 2 - - # Write BOM. - file_object.seek(0) - file_object.write(prefix) - # {read|write}_location separated for readability. - write_location = len(prefix) - - # Write and read block alternatingly. - while len(first_block): - # Write first block. - file_object.seek(write_location) - file_object.write(first_block) - # Increment write_location. - write_location += block_size - - # Move second block into first variable. - first_block = second_block - - # Read in the next block. - file_object.seek(read_location) - second_block = file_object.read(block_size) - # Increment read_location. - read_location += block_size diff --git a/pydrive2/fs/spec.py b/pydrive2/fs/spec.py index 880dea1..6166662 100644 --- a/pydrive2/fs/spec.py +++ b/pydrive2/fs/spec.py @@ -28,6 +28,8 @@ logger = logging.getLogger(__name__) FOLDER_MIME_TYPE = "application/vnd.google-apps.folder" +_INFO_FIELDS = "mimeType,size,md5Checksum" +_LIST_FIELDS = f"nextPageToken,files(id,name,parents,{_INFO_FIELDS})" _PARENT_QUERY_BATCH_SIZE = 50 COMMON_SETTINGS = { @@ -48,6 +50,18 @@ class GDriveAuthError(Exception): pass +def _file_entry(name, item): + if item["mimeType"] == FOLDER_MIME_TYPE: + return {"name": name.rstrip("/") + "/", "type": "directory", "size": 0} + size = item.get("size") + return { + "name": name, + "type": "file", + "size": int(size) if size is not None else None, + "checksum": item.get("md5Checksum"), + } + + def _quote_query_value(value): escaped = value.replace("\\", "\\\\").replace("'", "\\'") return f"'{escaped}'" @@ -289,8 +303,10 @@ def _cache_path_id(self, path, *item_ids, cache=None): @cached_property def _list_params(self): - params = {"corpora": "default"} - if self.root != "root" and self.root != "appDataFolder": + params = {"corpora": "user"} + if self.root == "appDataFolder": + params["spaces"] = "appDataFolder" + elif self.root != "root": drive_id = self._gdrive_shared_drive_id(self.root) if drive_id: logger.debug( @@ -320,7 +336,11 @@ def _gdrive_shared_drive_id(self, item_id): return item.get("driveId", None) def _gdrive_list(self, query): - param = {"q": query, "maxResults": 1000} + param = { + "q": query, + "pageSize": 1000, + "fields": _LIST_FIELDS, + } param.update(self._list_params) file_list = self.client.ListFile(param) @@ -330,32 +350,40 @@ def _gdrive_list(self, query): # Fetch pages until None is received, lazily flatten the thing. return chain.from_iterable(iter(get_list, None)) - def _gdrive_list_ids(self, query_ids, *, title=None): + def _gdrive_list_ids(self, query_ids, *, name=None): filters = "trashed=false" - if title is not None: - filters += f" and title={_quote_query_value(title)}" + if name is not None: + filters += f" and name={_quote_query_value(name)}" # Drive can reject large parent queries. Query each parent only once. + seen_ids = set() for batch in chunks(_PARENT_QUERY_BATCH_SIZE, distinct(query_ids)): parents = " or ".join( f"{_quote_query_value(parent)} in parents" for parent in batch ) - yield from self._gdrive_list(f"({parents}) and {filters}") + for item in self._gdrive_list(f"({parents}) and {filters}"): + file_id = item["id"] + if file_id in seen_ids: + continue + # Only a full batch can be followed by another parent query. + if len(batch) == _PARENT_QUERY_BATCH_SIZE: + seen_ids.add(file_id) + yield item def _get_remote_item_ids( - self, parent_ids, parent_path, title, use_cache=True + self, parent_ids, parent_path, name, use_cache=True ): if not parent_ids: return None res = [] - for item in self._gdrive_list_ids(parent_ids, title=title): + for item in self._gdrive_list_ids(parent_ids, name=name): # GDrive list API is case insensitive, we need to compare - # all results and pick the ones with the right title - if item["title"] == title: + # all results and pick the ones with the right name + if item["name"] == name: res.append(item["id"]) if item["mimeType"] == FOLDER_MIME_TYPE and use_cache: self._cache_path_id( - posixpath.join(parent_path, item["title"]), item["id"] + posixpath.join(parent_path, item["name"]), item["id"] ) return res @@ -373,16 +401,16 @@ def _path_to_item_ids(self, path, create=False, use_cache=True): if item_ids: return item_ids - parent_path, title = posixpath.split(path) + parent_path, name = posixpath.split(path) parent_ids = self._path_to_item_ids(parent_path, create, use_cache) item_ids = self._get_remote_item_ids( - parent_ids, parent_path, title, use_cache + parent_ids, parent_path, name, use_cache ) if item_ids: return item_ids return ( - [self._create_dir(min(parent_ids), title, path)] if create else [] + [self._create_dir(min(parent_ids), name, path)] if create else [] ) def _get_item_id(self, path, create=False, use_cache=True): @@ -397,21 +425,24 @@ def _get_item_id(self, path, create=False, use_cache=True): raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path) @_gdrive_retry - def _gdrive_create_dir(self, parent_id, title): - parent = {"id": parent_id} + def _gdrive_create_dir(self, parent_id, name): item = self.client.CreateFile( - {"title": title, "parents": [parent], "mimeType": FOLDER_MIME_TYPE} + { + "name": name, + "parents": [parent_id], + "mimeType": FOLDER_MIME_TYPE, + } ) item.Upload() return item @wrap_with(threading.RLock()) - def _create_dir(self, parent_id, title, remote_path): + def _create_dir(self, parent_id, name, remote_path): cached = self._ids_cache["dirs"].get(remote_path) if cached: return cached[0] - item = self._gdrive_create_dir(parent_id, title) + item = self._gdrive_create_dir(parent_id, name) self._cache_path_id(remote_path, item["id"]) return item["id"] @@ -429,18 +460,11 @@ def info(self, path): bucket, base = self.split_path(path) item_id = self._get_item_id(path) gdrive_file = self.client.CreateFile({"id": item_id}) - gdrive_file.FetchMetadata() + gdrive_file.FetchMetadata(fields=_INFO_FIELDS) - metadata = {"name": posixpath.join(bucket, base.rstrip("/"))} - if gdrive_file["mimeType"] == FOLDER_MIME_TYPE: - metadata["type"] = "directory" - metadata["size"] = 0 - metadata["name"] += "/" - else: - metadata["type"] = "file" - metadata["size"] = int(gdrive_file.get("fileSize")) - metadata["checksum"] = gdrive_file.get("md5Checksum") - return metadata + return _file_entry( + posixpath.join(bucket, base.rstrip("/")), gdrive_file + ) def ls(self, path, detail=False): bucket, base = self.split_path(path) @@ -455,25 +479,8 @@ def ls(self, path, detail=False): root_path = posixpath.join(bucket, base) contents = [] for item in self._gdrive_list_ids(dir_ids): - item_path = posixpath.join(root_path, item["title"]) - if item["mimeType"] == FOLDER_MIME_TYPE: - contents.append( - { - "type": "directory", - "name": item_path.rstrip("/") + "/", - "size": 0, - } - ) - else: - size = item.get("fileSize") - contents.append( - { - "type": "file", - "name": item_path, - "size": int(size) if size is not None else size, - "checksum": item.get("md5Checksum"), - } - ) + item_path = posixpath.join(root_path, item["name"]) + contents.append(_file_entry(item_path, item)) if detail: return contents @@ -506,20 +513,25 @@ def find(self, path, detail=False, **kwargs): new_query_ids: dict[str, str] = {} dir_ids.append(new_query_ids) for item in self._gdrive_list_ids(query_ids): - parent_id = item["parents"][0]["id"] - item_path = posixpath.join(query_ids[parent_id], item["title"]) + # A root alias in the query is returned as its canonical ID. + parent_id = next( + (p for p in item.get("parents", ()) if p in query_ids), + None, + ) + if parent_id is None: + if ( + self.root not in {"root", "appDataFolder"} + or self.root not in query_ids + ): + continue + parent_id = self.root + item_path = posixpath.join(query_ids[parent_id], item["name"]) if item["mimeType"] == FOLDER_MIME_TYPE: new_query_ids[item["id"]] = item_path self._cache_path_id(item_path, item["id"]) continue - size = item.get("fileSize") contents.append( - { - "name": posixpath.join(bucket, item_path), - "type": "file", - "size": int(size) if size is not None else size, - "checksum": item.get("md5Checksum"), - } + _file_entry(posixpath.join(bucket, item_path), item) ) if detail: @@ -544,22 +556,27 @@ def put_file(self, lpath, rpath, callback=None, **kwargs): self.upload_fobj(stream, rpath, callback=callback) @_gdrive_retry - def _gdrive_upload_fobj(self, title, parent_id, stream, callback=None): - item = self.client.CreateFile( - {"title": title, "parents": [{"id": parent_id}]} - ) + def _gdrive_upload_fobj(self, name, parent_id, stream, callback=None): + item = self.client.CreateFile({"name": name, "parents": [parent_id]}) item.content = stream item.Upload() return item def cp_file(self, lpath, rpath, **kwargs): - """In-memory streamed copy""" - with self.open(lpath) as stream: - # IterStream objects doesn't support full-length - # seek() calls, so we have to wrap the data with - # an external buffer. - buffer = io.BytesIO(stream.read()) - self.upload_fobj(buffer, rpath) + """Ask Drive to copy the file directly into the destination folder.""" + source_id = self._get_item_id(lpath) + parent_id = self._get_item_id(self._parent(rpath), create=True) + name = posixpath.basename(rpath.rstrip("/")) + self._gdrive_copy_file(source_id, parent_id, name) + + @_gdrive_retry + def _gdrive_copy_file(self, source_id, parent_id, name): + source = self.client.CreateFile({"id": source_id}) + source.Copy( + target_folder={"id": parent_id}, + new_title=name, + param={"fields": "id"}, + ) @_gdrive_retry def mv(self, path1, path2, maxdepth=None, **kwargs): @@ -582,11 +599,11 @@ def mv(self, path1, path2, maxdepth=None, **kwargs): file1 = self.client.CreateFile({"id": file1_id}) if src_name != dst_name: - file1["title"] = dst_name + file1["name"] = dst_name if src_parent != dst_parent: file2_parent_id = self._get_item_id(dst_parent) - file1["parents"] = [{"id": file2_parent_id}] + file1["parents"] = [file2_parent_id] # TODO need to invalidate the cache for the old path, see #232 file1.Upload() @@ -608,12 +625,15 @@ def _gdrive_get_file(self, item_id, rpath, callback=None, block_size=None): extra_args["chunksize"] = block_size if callback: + reported_size = None - def cb(value, _): + def cb(value, total): + nonlocal reported_size + if total is not None and total != reported_size: + callback.set_size(total) + reported_size = total callback.absolute_update(value) - gdrive_file.FetchMetadata(fields="fileSize") - callback.set_size(int(gdrive_file.get("fileSize"))) extra_args["callback"] = cb gdrive_file.GetContentFile(rpath, **extra_args) @@ -655,7 +675,10 @@ def _gdrive_delete_file(self, item_id): if ( http_error_code == 403 and self._list_params["corpora"] == "drive" - and exc.GetField("location") == "file.permissions" + and ( + exc.GetField("reason") == "insufficientFilePermissions" + or exc.GetField("location") == "file.permissions" + ) ): raise PermissionError( "Insufficient permissions to {}. You should have {} " diff --git a/pydrive2/test/README.rst b/pydrive2/test/README.rst index 8dd7be5..641affc 100644 --- a/pydrive2/test/README.rst +++ b/pydrive2/test/README.rst @@ -76,4 +76,4 @@ those tests locally, configure a service account as described below. .. _Google API Console: https://console.developers.google.com .. _Create a new: https://cloud.google.com/resource-manager/docs/creating-managing-projects#creating_a_project -.. _Enable the Drive API: https://developers.google.com/drive/api/v2/about-sdk +.. _Enable the Drive API: https://developers.google.com/workspace/drive/api/guides/about-sdk diff --git a/pydrive2/test/conftest.py b/pydrive2/test/conftest.py new file mode 100644 index 0000000..e60be63 --- /dev/null +++ b/pydrive2/test/conftest.py @@ -0,0 +1,19 @@ +import pytest +from google.oauth2.credentials import Credentials +from googleapiclient.http import HttpMockSequence + +from pydrive2.auth import GoogleAuth +from pydrive2.drive import GoogleDrive + + +@pytest.fixture +def mock_drive(): + def build(*responses): + http = HttpMockSequence(list(responses)) + auth = GoogleAuth(settings_file=None) + auth.credentials = Credentials("test-token") + auth.Authorize() + auth.thread_local.http = http + return GoogleDrive(auth), http.request_sequence + + return build diff --git a/pydrive2/test/http_mock.py b/pydrive2/test/http_mock.py new file mode 100644 index 0000000..19fceff --- /dev/null +++ b/pydrive2/test/http_mock.py @@ -0,0 +1,16 @@ +import json +from urllib.parse import parse_qs, urlsplit + + +def response(data): + return {"status": "200"}, json.dumps(data) + + +def error_response(code, reason): + return {"status": str(code)}, json.dumps( + {"error": {"code": code, "errors": [{"reason": reason}]}} + ) + + +def query(request): + return parse_qs(urlsplit(request[0]).query) diff --git a/pydrive2/test/test_drive_unit.py b/pydrive2/test/test_drive_unit.py new file mode 100644 index 0000000..8e42a33 --- /dev/null +++ b/pydrive2/test/test_drive_unit.py @@ -0,0 +1,41 @@ +from urllib.parse import urlsplit + +import pytest + +from pydrive2.test.http_mock import query, response + + +def test_list_retains_next_page_token(mock_drive): + drive, requests = mock_drive( + response({"nextPageToken": "more", "files": []}), + response({"files": [{"name": "second"}]}), + ) + listing = drive.ListFile({"fields": "files(name)"}) + assert [item["name"] for item in listing.GetList()] == ["second"] + first, second = requests + assert ( + urlsplit(first[0]).path + == urlsplit(second[0]).path + == "/drive/v3/files" + ) + assert query(first)["fields"] == ["nextPageToken,files(name)"] + assert query(second)["pageToken"] == ["more"] + + +def test_list_wildcard_includes_page_token(mock_drive): + drive, requests = mock_drive(response({"files": []})) + assert drive.ListFile({"fields": " * "}).GetList() == [] + assert query(requests[0])["fields"] == ["*"] + + +@pytest.mark.parametrize( + ("fields", "expected"), + [(None, "*"), ("user/displayName", "user/displayName")], +) +def test_about_fields(mock_drive, fields, expected): + about = {"user": {"displayName": "Drive User"}} + drive, requests = mock_drive(response(about)) + assert drive.GetAbout(fields=fields) == about + (request,) = requests + assert urlsplit(request[0]).path == "/drive/v3/about" + assert query(request)["fields"] == [expected] diff --git a/pydrive2/test/test_file.py b/pydrive2/test/test_file.py index adaed58..3e9fa43 100644 --- a/pydrive2/test/test_file.py +++ b/pydrive2/test/test_file.py @@ -4,7 +4,6 @@ import sys import unittest from concurrent.futures import ThreadPoolExecutor, as_completed -from io import BytesIO from pathlib import Path from tempfile import mkdtemp from time import time @@ -17,7 +16,7 @@ from pydrive2.auth import GoogleAuth from pydrive2.drive import GoogleDrive -from pydrive2.files import ApiRequestError, GoogleDriveFile +from pydrive2.files import ApiRequestError from pydrive2.test import test_util from pydrive2.test.test_util import ( create_file, @@ -55,7 +54,7 @@ def resource_key_request( ): if fetch_meta_data: # Fake meta data query response - content = json.dumps({"title": "N48E012.zip"}).encode() + content = json.dumps({"name": "N48E012.zip"}).encode() else: # Fake file content query response content = b"some content" @@ -86,7 +85,7 @@ def resource_key_request( class GoogleDriveFileTest(unittest.TestCase): """Tests basic file operations of files.GoogleDriveFile. Upload and download of contents and metadata, and thread-safety checks. - Equivalent to Files.insert, Files.update, Files.patch in Google Drive API. + Equivalent to Files.create and Files.update in Google Drive API. """ @classmethod @@ -115,12 +114,12 @@ def test_Files_Insert(self): drive = GoogleDrive(self.ga) file1 = drive.CreateFile() filename = self.getTempFile("firsttestfile") - file1["title"] = filename - pydrive_retry(file1.Upload) # Files.insert + file1["name"] = filename + pydrive_retry(file1.Upload) # Files.create - self.assertEqual(file1.metadata["title"], filename) + self.assertEqual(file1.metadata["name"], filename) file2 = drive.CreateFile({"id": file1["id"]}) # Download file from id. - self.assertEqual(file2["title"], filename) + self.assertEqual(file2["name"], filename) self.DeleteUploadedFiles(drive, [file1["id"]]) @@ -128,12 +127,12 @@ def test_Files_Insert_Unicode(self): drive = GoogleDrive(self.ga) file1 = drive.CreateFile() filename = self.getTempFile("첫번째 파일") - file1["title"] = filename - pydrive_retry(file1.Upload) # Files.insert + file1["name"] = filename + pydrive_retry(file1.Upload) # Files.create - self.assertEqual(file1.metadata["title"], filename) + self.assertEqual(file1.metadata["name"], filename) file2 = drive.CreateFile({"id": file1["id"]}) # Download file from id. - self.assertEqual(file2["title"], filename) + self.assertEqual(file2["name"], filename) self.DeleteUploadedFiles(drive, [file1["id"]]) @@ -142,22 +141,22 @@ def test_Files_Insert_Content_String(self): file1 = drive.CreateFile() filename = self.getTempFile("secondtestfile") content = "hello world!" - file1["title"] = filename + file1["name"] = filename file1.SetContentString(content) - pydrive_retry(file1.Upload) # Files.insert + pydrive_retry(file1.Upload) # Files.create self.assertEqual(file1.GetContentString(), content) pydrive_retry( file1.FetchContent ) # Force download and double check content - self.assertEqual(file1.metadata["title"], filename) + self.assertEqual(file1.metadata["name"], filename) self.assertEqual(file1.GetContentString(), content) file2 = drive.CreateFile({"id": file1["id"]}) # Download file from id. pydrive_retry(file2.FetchContent) self.assertEqual(file2.GetContentString(), content) - self.assertEqual(file2.metadata["title"], filename) + self.assertEqual(file2.metadata["name"], filename) self.DeleteUploadedFiles(drive, [file1["id"]]) @@ -166,12 +165,12 @@ def test_Files_Insert_Content_Unicode_String(self): file1 = drive.CreateFile() filename = self.getTempFile("두번째 파일") content = "안녕 세상아!" - file1["title"] = filename + file1["name"] = filename file1.SetContentString(content) - pydrive_retry(file1.Upload) # Files.insert + pydrive_retry(file1.Upload) # Files.create self.assertEqual(file1.GetContentString(), content) - self.assertEqual(file1.metadata["title"], filename) + self.assertEqual(file1.metadata["name"], filename) pydrive_retry( file1.FetchContent ) # Force download and double check content. @@ -180,7 +179,7 @@ def test_Files_Insert_Content_Unicode_String(self): file2 = drive.CreateFile({"id": file1["id"]}) # Download file from id. pydrive_retry(file2.FetchContent) self.assertEqual(file2.GetContentString(), content) - self.assertEqual(file2.metadata["title"], filename) + self.assertEqual(file2.metadata["name"], filename) self.DeleteUploadedFiles(drive, [file1["id"]]) @@ -189,11 +188,9 @@ def test_Files_Insert_Content_File(self): file1 = drive.CreateFile() contentFile = self.getTempFile("actual_content", "some string") file1.SetContentFile(contentFile) - pydrive_retry(file1.Upload) # Files.insert + pydrive_retry(file1.Upload) # Files.create - self.assertEqual( - file1.metadata["title"], os.path.basename(contentFile) - ) + self.assertEqual(file1.metadata["name"], os.path.basename(contentFile)) pydrive_retry( file1.FetchContent ) # Force download and double check content. @@ -213,17 +210,17 @@ def test_Files_Patch(self): file1 = drive.CreateFile() filename = self.getTempFile("prepatchtestfile") newfilename = self.getTempFile("patchtestfile") - file1["title"] = filename - pydrive_retry(file1.Upload) # Files.insert + file1["name"] = filename + pydrive_retry(file1.Upload) # Files.create - self.assertEqual(file1.metadata["title"], filename) - file1["title"] = newfilename - pydrive_retry(file1.Upload) # Files.patch + self.assertEqual(file1.metadata["name"], filename) + file1["name"] = newfilename + pydrive_retry(file1.Upload) # Files.update - self.assertEqual(file1.metadata["title"], newfilename) + self.assertEqual(file1.metadata["name"], newfilename) file2 = drive.CreateFile({"id": file1["id"]}) # Download file from id. pydrive_retry(file2.FetchMetadata) - self.assertEqual(file2.metadata["title"], newfilename) + self.assertEqual(file2.metadata["name"], newfilename) self.DeleteUploadedFiles(drive, [file1["id"]]) @@ -232,18 +229,18 @@ def test_Files_Patch_By_Id(self): file1 = drive.CreateFile() filename = self.getTempFile("prepatchtestfile") newfilename = self.getTempFile("patchtestfile") - file1["title"] = filename - pydrive_retry(file1.Upload) # Files.insert + file1["name"] = filename + pydrive_retry(file1.Upload) # Files.create file2 = drive.CreateFile( {"id": file1["id"]} ) # Patch file no download. - file2["title"] = newfilename - pydrive_retry(file2.Upload) # Files.patch + file2["name"] = newfilename + pydrive_retry(file2.Upload) # Files.update file3 = drive.CreateFile({"id": file1["id"]}) # Download file from id. pydrive_retry(file3.FetchMetadata) - self.assertEqual(file3.metadata["title"], newfilename) + self.assertEqual(file3.metadata["name"], newfilename) def test_Files_Patch_Skipping_Content(self): drive = GoogleDrive(self.ga) @@ -252,14 +249,14 @@ def test_Files_Patch_Skipping_Content(self): newfilename = self.getTempFile("patchtestfile") content = "hello world!" - file1["title"] = filename + file1["name"] = filename file1.SetContentString(content) - pydrive_retry(file1.Upload) # Files.insert - self.assertEqual(file1.metadata["title"], filename) + pydrive_retry(file1.Upload) # Files.create + self.assertEqual(file1.metadata["name"], filename) - file1["title"] = newfilename - pydrive_retry(file1.Upload) # Files.patch - self.assertEqual(file1.metadata["title"], newfilename) + file1["name"] = newfilename + pydrive_retry(file1.Upload) # Files.update + self.assertEqual(file1.metadata["name"], newfilename) self.assertEqual(file1.GetContentString(), content) self.DeleteUploadedFiles(drive, [file1["id"]]) @@ -272,10 +269,10 @@ def test_Files_Update_String(self): content = "hello world!" newcontent = "hello new world!" - file1["title"] = filename + file1["name"] = filename file1.SetContentString(content) - pydrive_retry(file1.Upload) # Files.insert - self.assertEqual(file1.metadata["title"], filename) + pydrive_retry(file1.Upload) # Files.create + self.assertEqual(file1.metadata["name"], filename) self.assertEqual(file1.GetContentString(), content) pydrive_retry( @@ -283,10 +280,10 @@ def test_Files_Update_String(self): ) # Force download and double check content. self.assertEqual(file1.GetContentString(), content) - file1["title"] = newfilename + file1["name"] = newfilename file1.SetContentString(newcontent) pydrive_retry(file1.Upload) # Files.update - self.assertEqual(file1.metadata["title"], newfilename) + self.assertEqual(file1.metadata["name"], newfilename) self.assertEqual(file1.GetContentString(), newcontent) self.DeleteUploadedFiles(drive, [file1["id"]]) @@ -299,10 +296,10 @@ def test_Files_Update_File(self): contentFile = self.getTempFile("actual_content", "some string") contentFile2 = self.getTempFile("actual_content_2", "some string") - file1["title"] = filename + file1["name"] = filename file1.SetContentFile(contentFile) - pydrive_retry(file1.Upload) # Files.insert - self.assertEqual(file1.metadata["title"], filename) + pydrive_retry(file1.Upload) # Files.create + self.assertEqual(file1.metadata["name"], filename) pydrive_retry( file1.FetchContent @@ -311,10 +308,10 @@ def test_Files_Update_File(self): pydrive_retry(file1.GetContentFile, fileOut) self.assertEqual(filecmp.cmp(contentFile, fileOut), True) - file1["title"] = newfilename + file1["name"] = newfilename file1.SetContentFile(contentFile2) pydrive_retry(file1.Upload) # Files.update - self.assertEqual(file1.metadata["title"], newfilename) + self.assertEqual(file1.metadata["name"], newfilename) fileOut = self.getTempFile() pydrive_retry(file1.GetContentFile, fileOut) @@ -330,15 +327,15 @@ def test_Files_Update_File_Keeps_Filename(self): contentFile = self.getTempFile("actual_content", "some string") contentFile2 = self.getTempFile("actual_content_2", "some string") - file1["title"] = filename + file1["name"] = filename file1.SetContentFile(contentFile) - pydrive_retry(file1.Upload) # Files.insert - self.assertEqual(file1.metadata["title"], filename) + pydrive_retry(file1.Upload) # Files.create + self.assertEqual(file1.metadata["name"], filename) same_file = drive.CreateFile({"id": file1["id"]}) same_file.SetContentFile(contentFile2) pydrive_retry(same_file.Upload) # Files.update - self.assertEqual(same_file.metadata["title"], filename) + self.assertEqual(same_file.metadata["name"], filename) self.DeleteUploadedFiles(drive, [file1["id"]]) @@ -350,19 +347,19 @@ def test_Files_Update_By_Id(self): content = "hello world!" newcontent = "hello new world!" - file1["title"] = filename + file1["name"] = filename file1.SetContentString(content) - pydrive_retry(file1.Upload) # Files.insert + pydrive_retry(file1.Upload) # Files.create file2 = drive.CreateFile({"id": file1["id"]}) - file2["title"] = newfilename + file2["name"] = newfilename file2.SetContentString(newcontent) pydrive_retry(file2.Upload) # Files.update file3 = drive.CreateFile({"id": file1["id"]}) pydrive_retry(file3.FetchContent) self.assertEqual(file3.GetContentString(), newcontent) - self.assertEqual(file3["title"], newfilename) + self.assertEqual(file3["name"], newfilename) def test_Files_Download_Service(self): """ @@ -374,10 +371,10 @@ def test_Files_Download_Service(self): filename = self.getTempFile("prepatchtestfile") content = "hello world!" - file1["title"] = filename + file1["name"] = filename file1.SetContentString(content) - pydrive_retry(file1.Upload) # Files.insert - self.assertEqual(file1.metadata["title"], filename) + pydrive_retry(file1.Upload) # Files.create + self.assertEqual(file1.metadata["name"], filename) fileOut1 = self.getTempFile() pydrive_retry(file1.GetContentFile, fileOut1) @@ -399,12 +396,12 @@ def test_Files_Get_Content_Buffer(self): file1 = drive.CreateFile() filename = self.getTempFile() content = "hello world!\ngoodbye, cruel world!" - file1["title"] = filename + file1["name"] = filename file1.SetContentString(content) - pydrive_retry(file1.Upload) # Files.insert + pydrive_retry(file1.Upload) # Files.create buffer1 = pydrive_retry(file1.GetContentIOBuffer) - self.assertEqual(file1.metadata["title"], filename) + self.assertEqual(file1.metadata["name"], filename) self.assertEqual(len(buffer1), len(content)) self.assertEqual(b"".join(iter(buffer1)).decode("ascii"), content) @@ -534,7 +531,7 @@ def test_Files_Fetch_Metadata_resourceKey(self): pydrive_retry(file1.FetchMetadata) - self.assertEqual(file1.metadata["title"], "N48E012.zip") + self.assertEqual(file1.metadata["name"], "N48E012.zip") def test_Upload_Download_Empty_File(self): filename = os.path.join(self.tmpdir, str(time())) @@ -577,18 +574,18 @@ def test_Files_Trash_File(self): drive = GoogleDrive(self.ga) file1 = drive.CreateFile() pydrive_retry(file1.Upload) - self.assertFalse(file1.metadata["labels"]["trashed"]) + self.assertFalse(file1.metadata["trashed"]) # Download to verify non-trashed state on GDrive. file2 = drive.CreateFile({"id": file1["id"]}) pydrive_retry(file2.FetchMetadata) - self.assertFalse(file2.metadata["labels"]["trashed"]) + self.assertFalse(file2.metadata["trashed"]) pydrive_retry(file1.Trash) - self.assertTrue(file1.metadata["labels"]["trashed"]) + self.assertTrue(file1.metadata["trashed"]) pydrive_retry(file2.FetchMetadata) - self.assertTrue(file2.metadata["labels"]["trashed"]) + self.assertTrue(file2.metadata["trashed"]) self.DeleteUploadedFiles(drive, [file1["id"]]) @@ -596,7 +593,7 @@ def test_Files_Trash_File_Just_ID(self): drive = GoogleDrive(self.ga) file1 = drive.CreateFile() pydrive_retry(file1.Upload) - self.assertFalse(file1.metadata["labels"]["trashed"]) + self.assertFalse(file1.metadata["trashed"]) # Trash file by ID. file2 = drive.CreateFile({"id": file1["id"]}) @@ -604,7 +601,7 @@ def test_Files_Trash_File_Just_ID(self): # Verify trashed by downloading metadata. pydrive_retry(file1.FetchMetadata) - self.assertTrue(file1.metadata["labels"]["trashed"]) + self.assertTrue(file1.metadata["trashed"]) self.DeleteUploadedFiles(drive, [file1["id"]]) @@ -613,20 +610,20 @@ def test_Files_UnTrash_File(self): file1 = drive.CreateFile() pydrive_retry(file1.Upload) pydrive_retry(file1.Trash) - self.assertTrue(file1.metadata["labels"]["trashed"]) + self.assertTrue(file1.metadata["trashed"]) # Verify that file is trashed by downloading metadata. file2 = drive.CreateFile({"id": file1["id"]}) pydrive_retry(file2.FetchMetadata) - self.assertTrue(file2.metadata["labels"]["trashed"]) + self.assertTrue(file2.metadata["trashed"]) # Un-trash the file, and assert local metadata is updated correctly. pydrive_retry(file1.UnTrash) - self.assertFalse(file1.metadata["labels"]["trashed"]) + self.assertFalse(file1.metadata["trashed"]) # Re-fetch the metadata, and assert file un-trashed on GDrive. pydrive_retry(file2.FetchMetadata) - self.assertFalse(file2.metadata["labels"]["trashed"]) + self.assertFalse(file2.metadata["trashed"]) self.DeleteUploadedFiles(drive, [file1["id"]]) @@ -635,13 +632,13 @@ def test_Files_UnTrash_File_Just_ID(self): file1 = drive.CreateFile() pydrive_retry(file1.Upload) pydrive_retry(file1.Trash) - self.assertTrue(file1.metadata["labels"]["trashed"]) + self.assertTrue(file1.metadata["trashed"]) file2 = drive.CreateFile({"id": file1["id"]}) pydrive_retry(file2.UnTrash) # UnTrash without fetching metadata. pydrive_retry(file1.FetchMetadata) - self.assertFalse(file1.metadata["labels"]["trashed"]) + self.assertFalse(file1.metadata["trashed"]) self.DeleteUploadedFiles(drive, [file1["id"]]) @@ -711,22 +708,22 @@ def test_Files_Insert_Permission(self): # Insert the permission. permission = pydrive_retry( file1.InsertPermission, - {"type": "anyone", "value": "anyone", "role": "reader"}, + {"type": "anyone", "role": "reader", "allowFileDiscovery": False}, ) self.assertTrue(permission) self.assertEqual(len(file1["permissions"]), 2) - self.assertEqual(file1["permissions"][0]["type"], "anyone") + self.assertIn("anyone", [p["type"] for p in file1["permissions"]]) permissions = pydrive_retry(file1.GetPermissions) self.assertEqual(len(file1["permissions"]), 2) - self.assertEqual(file1["permissions"][0]["type"], "anyone") - self.assertEqual(permissions[0]["type"], "anyone") + self.assertIn("anyone", [p["type"] for p in file1["permissions"]]) + self.assertIn("anyone", [p["type"] for p in permissions]) # Verify remote changes made. file2 = drive.CreateFile({"id": file1["id"]}) permissions = pydrive_retry(file2.GetPermissions) self.assertEqual(len(permissions), 2) - self.assertEqual(permissions[0]["type"], "anyone") + self.assertIn("anyone", [p["type"] for p in permissions]) pydrive_retry(file1.Delete) @@ -749,13 +746,14 @@ def test_Files_Delete_Permission(self): pydrive_retry(file1.Upload) pydrive_retry( file1.InsertPermission, - {"type": "anyone", "value": "anyone", "role": "reader"}, + {"type": "anyone", "role": "reader", "allowFileDiscovery": False}, ) permissions = pydrive_retry(file1.GetPermissions) self.assertEqual(len(permissions), 2) self.assertEqual(len(file1["permissions"]), 2) - pydrive_retry(file1.DeletePermission, permissions[0]["id"]) + public_id = next(p["id"] for p in permissions if p["type"] == "anyone") + pydrive_retry(file1.DeletePermission, public_id) self.assertEqual(len(file1["permissions"]), 1) # Verify remote changes made. @@ -814,17 +812,23 @@ def test_GFile_Conversion_Lossless_String(self): # Upload a string, and convert into Google Doc format. test_string = "Generic, non-exhaustive ASCII test string." file1.SetContentString(test_string) - pydrive_retry(file1.Upload, {"convert": True}) + file1["mimeType"] = "application/vnd.google-apps.document" + pydrive_retry(file1.Upload, media_mime_type="text/plain") + + self.addCleanup(pydrive_retry, file1.Delete) - # Download string as plain text. - downloaded_string = file1.GetContentString(mimetype="text/plain") + # A plain-text Google Docs export includes a UTF-8 BOM. + downloaded_string = file1.GetContentString( + mimetype="text/plain", remove_bom=True + ) self.assertEqual( test_string, downloaded_string, "Strings do not match" ) # Download content into file and ensure that file content matches original # content string. - downloaded_file_name = "_tmp_downloaded_file_name.txt" + downloaded_file_name = self.getTempFile() + self.addCleanup(Path(downloaded_file_name).unlink, missing_ok=True) pydrive_retry( file1.GetContentFile, downloaded_file_name, @@ -836,113 +840,71 @@ def test_GFile_Conversion_Lossless_String(self): test_string, downloaded_string, "Strings do not match" ) - # Delete temp file. - delete_file(downloaded_file_name) - # Tests for GDrive conversion. # ============================ - def setup_gfile_conversion_test(self): - drive = GoogleDrive(self.ga) - file1 = drive.CreateFile() - - # Create a file to upload. - file_name = "_tmp_source_file.txt" - downloaded_file_name = "_tmp_downloaded_file_name.txt" - original_file_content = "Generic, non-exhaustive\n ASCII test string." - Path(file_name).write_text(original_file_content) - original_file_content = test_util.StripNewlines(original_file_content) - - return file1, file_name, original_file_content, downloaded_file_name - - def cleanup_gfile_conversion_test( - self, file1, file_name, downloaded_file_name - ): - # Delete temporary files. - os.path.exists(file_name) and os.remove(file_name) - os.path.exists(downloaded_file_name) and os.remove( - downloaded_file_name + def upload_test_document(self): + content = "Generic, non-exhaustive\n ASCII test string." + source = self.getTempFile("source", content) + self.addCleanup(Path(source).unlink, missing_ok=True) + file = GoogleDrive(self.ga).CreateFile( + {"mimeType": "application/vnd.google-apps.document"} ) - pydrive_retry(file1.Delete) # Delete uploaded file. + file.SetContentFile(source) + with file.content: + pydrive_retry(file.Upload, media_mime_type="text/plain") + self.addCleanup(pydrive_retry, file.Delete) + return file, test_util.StripNewlines(content) def test_GFile_Conversion_Remove_BOM(self): - ( - file1, - file_name, - original_file_content, - downloaded_file_name, - ) = self.setup_gfile_conversion_test() - try: - # Upload source_file and convert into Google Doc format. - file1.SetContentFile(file_name) - pydrive_retry(file1.Upload, {"convert": True}) - - # Download as string. - downloaded_content_no_bom = file1.GetContentString( - mimetype="text/plain", remove_bom=True - ) - downloaded_content_no_bom = test_util.StripNewlines( - downloaded_content_no_bom - ) - self.assertEqual(original_file_content, downloaded_content_no_bom) - - # Download as file. - pydrive_retry( - file1.GetContentFile, downloaded_file_name, remove_bom=True - ) - downloaded_content = Path(downloaded_file_name).read_text() - downloaded_content = test_util.StripNewlines(downloaded_content) - self.assertEqual(original_file_content, downloaded_content) + file1, original_file_content = self.upload_test_document() + downloaded_file_name = self.getTempFile("export") + self.addCleanup(Path(downloaded_file_name).unlink, missing_ok=True) + # Download as string. + downloaded_content_no_bom = file1.GetContentString( + mimetype="text/plain", remove_bom=True + ) + downloaded_content_no_bom = test_util.StripNewlines( + downloaded_content_no_bom + ) + self.assertEqual(original_file_content, downloaded_content_no_bom) - finally: - self.cleanup_gfile_conversion_test( - file1, file_name, downloaded_file_name - ) + # Download as file. + pydrive_retry( + file1.GetContentFile, downloaded_file_name, remove_bom=True + ) + downloaded_content = Path(downloaded_file_name).read_text() + downloaded_content = test_util.StripNewlines(downloaded_content) + self.assertEqual(original_file_content, downloaded_content) def test_Gfile_Conversion_Add_Remove_BOM(self): - """Tests whether you can switch between the BOM appended and removed - version on the fly.""" - ( - file1, - file_name, - _, - downloaded_file_name, - ) = self.setup_gfile_conversion_test() - try: - file1.SetContentFile(file_name) - pydrive_retry(file1.Upload, {"convert": True}) - - content_bom = file1.GetContentString(mimetype="text/plain") - content_no_bom = file1.GetContentString( - mimetype="text/plain", remove_bom=True - ) - content_bom_2 = file1.GetContentString(mimetype="text/plain") + """Switch between the exported text with and without the BOM.""" + file1, _ = self.upload_test_document() + content_bom = file1.GetContentString(mimetype="text/plain") + content_no_bom = file1.GetContentString( + mimetype="text/plain", remove_bom=True + ) + content_bom_2 = file1.GetContentString(mimetype="text/plain") - self.assertEqual(content_bom, content_bom_2) - self.assertNotEqual(content_bom, content_no_bom) - self.assertTrue(len(content_bom) > len(content_no_bom)) + self.assertEqual(content_bom, content_bom_2) + self.assertEqual(content_bom, "\ufeff" + content_no_bom) - buffer_bom = pydrive_retry( - file1.GetContentIOBuffer, - mimetype="text/plain", - encoding="utf-8", - ) - buffer_bom = "".join(iter(buffer_bom)) - self.assertEqual(content_bom, buffer_bom) - - buffer_no_bom = pydrive_retry( - file1.GetContentIOBuffer, - mimetype="text/plain", - remove_bom=True, - encoding="utf-8", - ) - buffer_no_bom = "".join(iter(buffer_no_bom)) - self.assertEqual(content_no_bom, buffer_no_bom) + buffer_bom = pydrive_retry( + file1.GetContentIOBuffer, + mimetype="text/plain", + encoding="utf-8", + ) + buffer_bom = "".join(iter(buffer_bom)) + self.assertEqual(content_bom, buffer_bom) - finally: - self.cleanup_gfile_conversion_test( - file1, file_name, downloaded_file_name - ) + buffer_no_bom = pydrive_retry( + file1.GetContentIOBuffer, + mimetype="text/plain", + remove_bom=True, + encoding="utf-8", + ) + buffer_no_bom = "".join(iter(buffer_no_bom)) + self.assertEqual(content_no_bom, buffer_no_bom) # Setup for concurrent upload testing. # ===================================== @@ -963,14 +925,14 @@ def _parallel_uploader(self, num_of_uploads, num_of_workers): for i in range(num_of_uploads): file_name = first_file if i % 2 == 0 else second_file up_file = drive.CreateFile() - up_file["title"] = remote_name + up_file["name"] = remote_name up_file.SetContentFile(file_name) upload_files.append(up_file) # Ensure there are no files with the random file name. files = pydrive_retry( lambda: drive.ListFile( - param={"q": f"title = '{remote_name}' and trashed = false"} + param={"q": f"name = '{remote_name}' and trashed = false"} ).GetList() ) self.assertTrue(len(files) == 0) @@ -991,7 +953,7 @@ def _parallel_uploader(self, num_of_uploads, num_of_workers): def check_uploaded_files(): # A successful upload can take time to appear in title searches. files = drive.ListFile( - param={"q": f"title = '{remote_name}' and trashed = false"} + param={"q": f"name = '{remote_name}' and trashed = false"} ).GetList() if len(files) < num_of_uploads: raise test_util.PyDriveRetriableError( @@ -1010,19 +972,19 @@ def _parallel_downloader(self, file_ids, num_of_workers): download_files = [] for file_id in file_ids: file1 = drive.CreateFile({"id": file_id}) - file1["title"] = self.getTempFile() + file1["name"] = self.getTempFile() download_files.append(file1) # Ensure files don't exist yet. for file_obj in download_files: - self.assertTrue(not delete_file(file_obj["title"])) + self.assertTrue(not delete_file(file_obj["name"])) # Submit upload jobs to ThreadPoolExecutor. futures = [] for file_obj in download_files: futures.append( thread_pool.submit( - pydrive_retry, file_obj.GetContentFile, file_obj["title"] + pydrive_retry, file_obj.GetContentFile, file_obj["name"] ) ) @@ -1033,7 +995,7 @@ def _parallel_downloader(self, file_ids, num_of_workers): # Ensure all files were downloaded. for file_obj in download_files: - self.assertTrue(delete_file(file_obj["title"])) + self.assertTrue(delete_file(file_obj["name"])) # Remove uploaded files. self.DeleteUploadedFiles(drive, file_ids) @@ -1058,7 +1020,7 @@ def test_CopyFileSameFolder(self): # create a temp file and set it's content to a known string file1 = drive.CreateFile() filename = self.getTempFile("copytestfile", content=content) - file1["title"] = filename + file1["name"] = filename file1.SetContentFile(filename) pydrive_retry(file1.Upload) @@ -1081,7 +1043,7 @@ def test_CopyFileDifferentFolder(self): # create a temp file and set it's content to a known string file1 = drive.CreateFile() filename = self.getTempFile("copytestfile", content=content) - file1["title"] = filename + file1["name"] = filename file1.SetContentFile(filename) pydrive_retry(file1.Upload) @@ -1089,9 +1051,9 @@ def test_CopyFileDifferentFolder(self): temp_dir = pydrive_retry( drive.CreateFile, { - "title": "temp_dir", + "name": "temp_dir", "mimeType": "application/vnd.google-apps.folder", - "parents": [{"id": file1["parents"][0]["id"]}], + "parents": [file1["parents"][0]], }, ) pydrive_retry(temp_dir.Upload) @@ -1110,7 +1072,7 @@ def test_CopyFileDifferentFolder(self): files = pydrive_retry( drive.ListFile, {"q": f"'{temp_dir['id']}' in parents"} ).GetList() - self.assertIn("copytestfile_copy", [f["title"] for f in files]) + self.assertIn("copytestfile_copy", [f["name"] for f in files]) self.DeleteUploadedFiles( drive, [file1["id"], file2["id"], temp_dir["id"]] @@ -1125,58 +1087,5 @@ def DeleteUploadedFiles(self, drive, ids): pydrive_retry(tmp_file.Delete) -class FilePrefixTest(unittest.TestCase): - def test_InsertPrefix(self): - # Create BytesIO. - file_obj = BytesIO(b"abc") - original_length = len(file_obj.getvalue()) - char_to_insert = "\ufeff".encode() - - # Insert the prefix. - GoogleDriveFile._InsertPrefix(file_obj, char_to_insert) - modified_length = len(file_obj.getvalue()) - self.assertGreater(modified_length, original_length) - self.assertEqual(file_obj.getvalue(), "\ufeffabc".encode()) - - def test_InsertPrefixLarge(self): - # Create BytesIO. - test_content = "abc" * 800 - file_obj = BytesIO(test_content.encode("utf-8")) - original_length = len(file_obj.getvalue()) - char_to_insert = "\ufeff".encode() - - # Insert the prefix. - GoogleDriveFile._InsertPrefix(file_obj, char_to_insert) - modified_length = len(file_obj.getvalue()) - self.assertGreater(modified_length, original_length) - expected_content = "\ufeff" + test_content - self.assertEqual(file_obj.getvalue(), expected_content.encode("utf8")) - - def test_RemovePrefix(self): - # Create BytesIO. - file_obj = BytesIO("\ufeffabc".encode()) - original_length = len(file_obj.getvalue()) - char_to_remove = "\ufeff".encode() - - # Insert the prefix. - GoogleDriveFile._RemovePrefix(file_obj, char_to_remove) - modified_length = len(file_obj.getvalue()) - self.assertLess(modified_length, original_length) - self.assertEqual(file_obj.getvalue(), b"abc") - - def test_RemovePrefixLarge(self): - # Create BytesIO. - test_content = "\ufeff" + "abc" * 800 - file_obj = BytesIO(test_content.encode("utf8")) - original_length = len(file_obj.getvalue()) - char_to_remove = "\ufeff".encode() - - # Insert the prefix. - GoogleDriveFile._RemovePrefix(file_obj, char_to_remove) - modified_length = len(file_obj.getvalue()) - self.assertLess(modified_length, original_length) - self.assertEqual(file_obj.getvalue(), test_content[1:].encode("utf8")) - - if __name__ == "__main__": unittest.main() diff --git a/pydrive2/test/test_filelist.py b/pydrive2/test/test_filelist.py index 4bfb2f2..3838630 100644 --- a/pydrive2/test/test_filelist.py +++ b/pydrive2/test/test_filelist.py @@ -30,14 +30,14 @@ def setUpClass(cls): cls.title = test_util.CreateRandomFileName() cls.file_list = [] for _ in range(10): - file = cls.drive.CreateFile({"title": cls.title}) + file = cls.drive.CreateFile({"name": cls.title}) pydrive_retry(file.Upload) cls.addClassCleanup(pydrive_retry, file.Delete) cls.file_list.append(file) def test_01_Files_List_GetList(self): drive = GoogleDrive(self.ga) - query = f"title = '{self.title}' and trashed = false" + query = f"name = '{self.title}' and trashed = false" for file1 in pydrive_list_item(drive, query): found = False for file2 in pydrive_list_item(drive, query): @@ -47,7 +47,7 @@ def test_01_Files_List_GetList(self): def test_02_Files_List_ForLoop(self): drive = GoogleDrive(self.ga) - query = f"title = '{self.title}' and trashed = false" + query = f"name = '{self.title}' and trashed = false" files = list(pydrive_list_item(drive, query, 2)) for file1 in self.file_list: found = False @@ -60,8 +60,8 @@ def test_03_Files_List_GetList_Iterate(self): drive = GoogleDrive(self.ga) flist = drive.ListFile( { - "q": f"title = '{self.title}' and trashed = false", - "maxResults": 2, + "q": f"name = '{self.title}' and trashed = false", + "pageSize": 2, } ) files = [] @@ -84,12 +84,12 @@ def test_File_List_Folders(self): folder1 = drive.CreateFile( { "mimeType": "application/vnd.google-apps.folder", - "title": self.title, + "name": self.title, } ) pydrive_retry(folder1.Upload) self.addCleanup(pydrive_retry, folder1.Delete) - query = f"title = '{self.title}' and trashed = false" + query = f"name = '{self.title}' and trashed = false" self.assertCountEqual( [file["id"] for file in pydrive_list_item(drive, query)], [file["id"] for file in self.file_list] + [folder1["id"]], diff --git a/pydrive2/test/test_files_unit.py b/pydrive2/test/test_files_unit.py new file mode 100644 index 0000000..1577bec --- /dev/null +++ b/pydrive2/test/test_files_unit.py @@ -0,0 +1,449 @@ +import gc +import json +import weakref +from codecs import BOM_UTF8 +from urllib.parse import urlsplit + +import pytest +from googleapiclient.http import HttpMockSequence + +from pydrive2.files import ( + DEFAULT_FILE_FIELDS, + ApiRequestError, + FileNotDownloadableError, + FileNotUploadedError, +) +from pydrive2.test.http_mock import error_response, query, response + + +@pytest.mark.parametrize("kind", ["folder", "shortcut"]) +def test_get_content_of_non_downloadable_file(mock_drive, kind): + drive, requests = mock_drive( + response( + {"id": "file", "mimeType": f"application/vnd.google-apps.{kind}"} + ) + ) + with pytest.raises(FileNotDownloadableError): + drive.CreateFile({"id": "file"}).GetContentString() + assert len(requests) == 1 + + +def test_cached_workspace_content_uses_export(mock_drive): + metadata = { + "id": "doc", + "mimeType": "application/vnd.google-apps.document", + } + drive, requests = mock_drive( + response({"files": [metadata]}), ({"status": "200"}, b"document") + ) + (file,) = drive.ListFile().GetList() + assert file.GetContentString() == "document" + listing, export = requests + assert urlsplit(listing[0]).path.endswith("/files") + assert urlsplit(export[0]).path.endswith("/files/doc/export") + + +@pytest.mark.parametrize( + ("code", "reason", "exception"), + [ + (403, "fileNotExportable", FileNotDownloadableError), + (400, "conversionUnsupported", FileNotDownloadableError), + (403, "insufficientFilePermissions", ApiRequestError), + ], +) +def test_export_errors(mock_drive, code, reason, exception): + drive, _ = mock_drive( + response( + {"id": "form", "mimeType": "application/vnd.google-apps.form"} + ), + error_response(403, "fileNotDownloadable"), + error_response(code, reason), + ) + with pytest.raises(exception): + drive.CreateFile({"id": "form"}).GetContentString() + + +@pytest.mark.parametrize( + ("fields", "expected"), + [ + ("permissions(role,id,type)", "id,permissions(role,id,type)"), + ( + "permissions(permissionDetails(role,permissionType),id), id", + "permissions(permissionDetails(role,permissionType),id), id", + ), + (" * ", "*"), + (None, DEFAULT_FILE_FIELDS), + ], +) +def test_create_retains_id_for_next_upload(mock_drive, fields, expected): + drive, requests = mock_drive( + response({"id": "new"}), response({"id": "new"}) + ) + file = drive.CreateFile({"name": "first", "parents": ["folder"]}) + file.Upload({"fields": fields}) + file["name"] = "second" + file.Upload() + create, update = requests + assert create[1] == "POST" and query(create)["fields"] == [expected] + assert json.loads(create[2]) == {"name": "first", "parents": ["folder"]} + assert update[1] == "PATCH" and urlsplit(update[0]).path.endswith( + "/files/new" + ) + assert json.loads(update[2]) == {"name": "second"} + + +def test_upload_rejects_listed_file_without_id(mock_drive): + drive, requests = mock_drive( + response({"files": [{"name": "original"}]}), + response({"id": "duplicate"}), + ) + (file,) = drive.ListFile({"fields": "files(name)"}).GetList() + file["name"] = "renamed" + with pytest.raises(FileNotUploadedError, match="Drive file ID"): + file.Upload() + assert [request[1] for request in requests] == ["GET"] + + +def test_upload_updates_listed_file_with_id(mock_drive): + drive, requests = mock_drive( + response({"files": [{"id": "existing", "name": "original"}]}), + response({"id": "existing", "name": "renamed"}), + ) + (file,) = drive.ListFile({"fields": "files(id,name)"}).GetList() + file["name"] = "renamed" + file.Upload() + listing, update = requests + assert listing[1] == "GET" + assert update[1] == "PATCH" + assert urlsplit(update[0]).path == "/drive/v3/files/existing" + assert json.loads(update[2]) == {"name": "renamed"} + + +def test_move_fetches_current_parent_after_projected_update(mock_drive): + drive, requests = mock_drive( + response({"parents": ["A"]}), + response({"id": "file"}), + response({"parents": ["B"]}), + response({"parents": ["C"]}), + ) + file = drive.CreateFile({"id": "file"}) + file.FetchMetadata() + file.Upload({"addParents": "B", "removeParents": "A", "fields": "id"}) + file["parents"] = ["C"] + file.Upload() + _, _, fetch_parents, move = requests + assert query(fetch_parents)["fields"] == ["parents"] + assert query(move)["removeParents"] == ["B"] + assert query(move)["addParents"] == ["C"] + assert "parents" not in json.loads(move[2]) + + +def test_move_to_existing_root_by_alias(mock_drive): + drive, requests = mock_drive( + response({"parents": ["canonical-root"]}), + response({"id": "canonical-root"}), + response({}), + ) + file = drive.CreateFile({"id": "file", "parents": ["root"]}) + file.Upload() + fetch, root, update = requests + assert query(fetch)["fields"] == ["parents"] + assert urlsplit(root[0]).path.endswith("/files/root") + assert not {"addParents", "removeParents"}.intersection(query(update)) + + +@pytest.mark.parametrize( + "metadata", [{"name": "new"}, {"id": "existing"}], ids=["create", "update"] +) +def test_upload_rejects_multiple_parents(mock_drive, metadata): + drive, requests = mock_drive() + with pytest.raises(ValueError, match="one parent"): + drive.CreateFile(dict(metadata, parents=["A", "B"])).Upload() + assert not requests + + +def test_custom_transport_is_used_without_mutating_params(mock_drive): + drive, default_requests = mock_drive() + http = HttpMockSequence( + [ + response({"id": "new"}), + response({}), + response({}), + response({}), + ({"status": "204"}, ""), + ] + ) + params = {"http": http} + file = drive.CreateFile({"name": "test"}) + file.Upload(params) + file.Upload(param=params) + file.Trash(param=params) + assert file["trashed"] is True + file.UnTrash(params) + assert file["trashed"] is False + file.Delete(params) + create, update, trash, untrash, delete = http.request_sequence + assert [r[1] for r in (create, update, trash, untrash, delete)] == [ + "POST", + "PATCH", + "PATCH", + "PATCH", + "DELETE", + ] + assert json.loads(trash[2]) == {"trashed": True} + assert json.loads(untrash[2]) == {"trashed": False} + assert params == {"http": http} and not default_requests + + +def test_copy_sets_name_and_parent(mock_drive): + drive, requests = mock_drive(response({"id": "copy"})) + copy = drive.CreateFile({"id": "source"}).Copy( + target_folder={"id": "destination"}, new_title="second" + ) + assert copy.uploaded and copy["id"] == "copy" + (request,) = requests + assert urlsplit(request[0]).path.endswith("/files/source/copy") + assert json.loads(request[2]) == { + "name": "second", + "parents": ["destination"], + } + + +def test_workspace_import_uses_source_media_type(mock_drive): + target = "application/vnd.google-apps.spreadsheet" + drive, requests = mock_drive( + ({"status": "200", "location": "https://example.test/session"}, ""), + response({"id": "sheet", "mimeType": target}), + ) + file = drive.CreateFile({"name": "sales", "mimeType": target}) + file.SetContentString("year,total\n2026,10") + file.Upload(media_mime_type="text/csv") + create, media = requests + assert urlsplit(create[0]).path == "/upload/drive/v3/files" + assert json.loads(create[2])["mimeType"] == target + assert {k.lower(): v for k, v in create[3].items()}[ + "x-upload-content-type" + ] == "text/csv" + assert media[1] == "PUT" + assert file["id"] == "sheet" and file.content is None + + +def test_id_only_workspace_update_discards_source_bytes(mock_drive): + target = "application/vnd.google-apps.document" + drive, requests = mock_drive( + ({"status": "200", "location": "https://example.test/session"}, ""), + response({"id": "doc", "mimeType": target}), + ) + file = drive.CreateFile({"id": "doc"}) + file.SetContentString("source") + file.Upload({"fields": "id"}, media_mime_type="text/plain") + assert file.content is None + assert query(requests[0])["fields"] == ["mimeType,id"] + + +def test_insert_permission_respects_projection_and_refreshes_cache(mock_drive): + permission = { + "role": "reader", + "type": "user", + "emailAddress": "reader@example.test", + } + saved = dict(permission, id="reader") + drive, requests = mock_drive( + response(saved), response({"permissions": [saved]}) + ) + file = drive.CreateFile({"id": "file"}) + assert file.InsertPermission(permission) == saved + assert file["permissions"] == [saved] + create, refresh = requests + assert query(create)["fields"] == ["*"] + assert json.loads(create[2]) == permission + assert refresh[1] == "GET" + + +def test_permissions_paginate_and_delete_from_cache(mock_drive): + reader, owner = {"id": "reader"}, {"id": "owner"} + drive, requests = mock_drive( + response({"permissions": [reader], "nextPageToken": "more"}), + response({"permissions": [owner]}), + ({"status": "204"}, ""), + ) + file = drive.CreateFile({"id": "file"}) + assert file.GetPermissions() == [reader, owner] + assert file.DeletePermission("reader") + assert file["permissions"] == [owner] + _, next_page, delete = requests + assert query(next_page)["pageToken"] == ["more"] + assert urlsplit(delete[0]).path.endswith("/permissions/reader") + + +def test_revisions_pagination(mock_drive): + drive, requests = mock_drive( + response({"revisions": [{"id": "1"}], "nextPageToken": "more"}), + response({"revisions": [{"id": "2"}]}), + ) + assert drive.CreateFile({"id": "file"}).GetRevisions() == [ + {"id": "1"}, + {"id": "2"}, + ] + first, second = requests + assert urlsplit(first[0]).path.endswith("/files/file/revisions") + assert query(second)["pageToken"] == ["more"] + + +@pytest.mark.parametrize("operation", ["GetPermissions", "GetRevisions"]) +def test_child_list_errors_retain_drive_reason(mock_drive, operation): + drive, _ = mock_drive(error_response(403, "rateLimitExceeded")) + with pytest.raises(ApiRequestError) as caught: + getattr(drive.CreateFile({"id": "file"}), operation)() + assert caught.value.GetField("reason") == "rateLimitExceeded" + + +def read_stream(file, tmp_path, **kwargs): + return b"".join(file.GetContentIOBuffer(**kwargs)) + + +def read_disk(file, tmp_path, **kwargs): + dest = tmp_path / "document" + file.GetContentFile(dest, **kwargs) + return dest.read_bytes() + + +def read_string(file, tmp_path, **kwargs): + return file.GetContentString(**kwargs).encode() + + +@pytest.mark.parametrize("read", [read_disk, read_stream, read_string]) +@pytest.mark.parametrize( + ("source", "target"), + [ + ("document", "text/plain"), + ("spreadsheet", "text/csv"), + ("presentation", "text/plain"), + ("drawing", "image/svg+xml"), + ], +) +def test_supplied_workspace_type_goes_directly_to_export( + mock_drive, tmp_path, read, source, target +): + drive, requests = mock_drive(({"status": "200"}, b"export")) + file = drive.CreateFile( + {"id": "doc", "mimeType": f"application/vnd.google-apps.{source}"} + ) + assert read(file, tmp_path, mimetype=target) == b"export" + (export,) = requests + assert urlsplit(export[0]).path == "/drive/v3/files/doc/export" + assert query(export)["mimeType"] == [target] + + +@pytest.mark.parametrize("read", [read_stream, read_disk]) +@pytest.mark.parametrize("known_type", [True, False], ids=["known", "id-only"]) +def test_workspace_export_requests(mock_drive, read, known_type, tmp_path): + body = BOM_UTF8 + b"text" + responses = [({"status": "200", "content-length": str(len(body))}, body)] + metadata = {"id": "doc", "resourceKey": "key"} + if known_type: + metadata["mimeType"] = "application/vnd.google-apps.document" + else: + responses.insert(0, error_response(403, "fileNotDownloadable")) + drive, requests = mock_drive(*responses) + assert ( + read(drive.CreateFile(metadata), tmp_path, remove_bom=True) == b"text" + ) + assert len(requests) == (1 if known_type else 2) + assert urlsplit(requests[-1][0]).path.endswith("/doc/export") + assert all( + r[3]["X-Goog-Drive-Resource-Keys"] == "doc/key" for r in requests + ) + + +@pytest.mark.parametrize("read", [read_stream, read_disk]) +def test_plain_text_media_removes_bom_across_chunks( + mock_drive, read, tmp_path +): + drive, _ = mock_drive( + ({"status": "206", "content-range": "bytes 0-1/5"}, BOM_UTF8[:2]), + ( + {"status": "206", "content-range": "bytes 2-3/5"}, + BOM_UTF8[2:] + b"x", + ), + ({"status": "206", "content-range": "bytes 4-4/5"}, b"y"), + ) + file = drive.CreateFile({"id": "blob"}) + assert ( + read( + file, tmp_path, mimetype="text/plain", remove_bom=True, chunksize=2 + ) + == b"xy" + ) + + +def test_stream_decodes_utf8_across_chunks(mock_drive): + drive, _ = mock_drive( + ({"status": "206", "content-range": "bytes 0-0/2"}, b"\xc3"), + ({"status": "206", "content-range": "bytes 1-1/2"}, b"\xa9"), + ) + file = drive.CreateFile({"id": "blob"}) + assert ( + "".join(file.GetContentIOBuffer(encoding="utf-8", chunksize=1)) == "é" + ) + + +@pytest.mark.parametrize( + "denied", + [ + error_response(403, "insufficientFilePermissions"), + ({"status": "403"}, '{"error":{"errors":[]}}'), + ({"status": "503"}, "unavailable"), + ], +) +def test_first_download_error_preserves_existing_file( + mock_drive, tmp_path, denied +): + drive, requests = mock_drive(denied) + dest = tmp_path / "existing" + dest.write_bytes(b"previous contents") + with pytest.raises(ApiRequestError) as caught: + drive.CreateFile({"id": "private"}).GetContentFile(dest) + assert caught.value.error["code"] == int(denied[0]["status"]) + assert dest.read_bytes() == b"previous contents" + assert len(requests) == 1 + + +def test_later_download_error_leaves_completed_chunks(mock_drive, tmp_path): + drive, _ = mock_drive( + ({"status": "206", "content-range": "bytes 0-1/4"}, b"ab"), + error_response(503, "backendError"), + ) + dest = tmp_path / "existing" + dest.write_bytes(b"previous contents") + with pytest.raises(ApiRequestError): + drive.CreateFile({"id": "file"}).GetContentFile(dest, chunksize=2) + assert dest.read_bytes() == b"ab" + + +def test_disk_download_releases_each_chunk_before_requesting_the_next( + mock_drive, tmp_path, monkeypatch +): + class Chunk(bytearray): + """A response body whose lifetime can be observed with a weak reference.""" + + first = Chunk(b"ab") + previous = weakref.ref(first) + drive, _ = mock_drive( + ({"status": "206", "content-range": "bytes 0-1/4"}, first), + ({"status": "206", "content-range": "bytes 2-3/4"}, b"cd"), + ) + del first + transport = drive.auth.thread_local.http + request = transport.request + + def check_request(uri, method, **kwargs): + if kwargs["headers"]["range"] == "bytes=2-3": + gc.collect() + assert previous() is None + return request(uri, method, **kwargs) + + monkeypatch.setattr(transport, "request", check_request) + dest = tmp_path / "download" + drive.CreateFile({"id": "file"}).GetContentFile(dest, chunksize=2) + assert dest.read_bytes() == b"abcd" diff --git a/pydrive2/test/test_fs.py b/pydrive2/test/test_fs.py index 73054d5..e013fa8 100644 --- a/pydrive2/test/test_fs.py +++ b/pydrive2/test/test_fs.py @@ -131,6 +131,15 @@ def test_move(fs, remote_dir): assert initial_info == secondary_info +def test_copy_file_to_new_directory(fs, remote_dir): + source = remote_dir + "/source" + target = remote_dir + "/nested/copy" + fs.pipe_file(source, b"payload") + fs.cp_file(source, target) + assert fs.cat_file(target) == b"payload" + assert fs.cat_file(source) == b"payload" + + def test_rm(fs, remote_dir): fs.touch(remote_dir + "/a.txt") fs.rm(remote_dir + "/a.txt") diff --git a/pydrive2/test/test_fs_listing.py b/pydrive2/test/test_fs_listing.py index 9ab5dd3..b97fb33 100644 --- a/pydrive2/test/test_fs_listing.py +++ b/pydrive2/test/test_fs_listing.py @@ -26,11 +26,13 @@ def list_query(query): assert 0 < len(parents) <= 50 assert query.endswith(") and trashed=false") listed.extend(parents) - return iter(parents) + return ({"id": parent} for parent in parents) monkeypatch.setattr(listing_fs, "_gdrive_list", list_query) - assert list(listing_fs._gdrive_list_ids(container(ids))) == ids + assert [ + item["id"] for item in listing_fs._gdrive_list_ids(container(ids)) + ] == ids assert listed == ids @@ -42,17 +44,22 @@ def parents(): consumed.append(index) yield str(index) - list_query = Mock(side_effect=[iter(["page-1", "page-2"]), iter(["last"])]) + list_query = Mock( + side_effect=[ + iter([{"id": "page-1"}, {"id": "page-2"}]), + iter([{"id": "last"}]), + ] + ) monkeypatch.setattr(listing_fs, "_gdrive_list", list_query) result = listing_fs._gdrive_list_ids(parents()) assert consumed == [] list_query.assert_not_called() - assert next(result) == "page-1" + assert next(result)["id"] == "page-1" assert len(consumed) == 50 - assert next(result) == "page-2" + assert next(result)["id"] == "page-2" assert list_query.call_count == 1 - assert list(result) == ["last"] + assert list(result) == [{"id": "last"}] assert len(consumed) == 51 assert list_query.call_count == 2 @@ -65,11 +72,11 @@ def test_ls_repeated_cached_parent_across_batches(listing_fs, monkeypatch): def list_file(params): batch = re.findall(r"'([^']+)' in parents", params["q"]) batches.append(batch) - # Distinct children with the same title must all remain in the listing. + # Distinct children with the same name must all remain in the listing. children = [ { "id": f"child-{parent}", - "title": "file", + "name": "file", "mimeType": "text/plain", } for parent in dict.fromkeys(batch) @@ -90,7 +97,13 @@ def list_query(query): # Simulate another caller caching a parent during the first batch. listing_fs._cache_path_id("dir", "late-parent") if "'late-parent' in parents" in query: - return [{"title": "late-file", "mimeType": "text/plain"}] + return [ + { + "id": "late-file", + "name": "late-file", + "mimeType": "text/plain", + } + ] return [] monkeypatch.setattr(listing_fs, "_gdrive_list", list_query) @@ -101,30 +114,30 @@ def list_query(query): def test_path_lookup_batches_duplicate_folder_names(listing_fs, monkeypatch): parents = [f"id-{i}" for i in range(51)] listing_fs._cache_path_id("duplicate", *parents) - title = r"File's\name" + name = r"File's\name" batches = [] def list_file(params): query = params["q"] batch = re.findall(r"'([^']+)' in parents", query) assert 0 < len(batch) <= 50 - assert r"title='File\'s\\name'" in query + assert r"name='File\'s\\name'" in query assert "trashed=false" in query batches.append(batch) if parents[-1] in batch: - item_id, item_title = "found", title + item_id, item_name = "found", name else: - item_id, item_title = "wrong-case", title.lower() + item_id, item_name = "wrong-case", name.lower() item = { "id": item_id, - "title": item_title, + "name": item_name, "mimeType": FOLDER_MIME_TYPE, } return iter([[item]]) monkeypatch.setattr(listing_fs.client, "ListFile", list_file) - assert listing_fs.exists(f"root/duplicate/{title}") - assert listing_fs._ids_cache["dirs"][f"duplicate/{title}"] == ["found"] + assert listing_fs.exists(f"root/duplicate/{name}") + assert listing_fs._ids_cache["dirs"][f"duplicate/{name}"] == ["found"] assert batches == [parents[:50], parents[50:]] @@ -139,18 +152,19 @@ def list_query(query): return iter( { "id": folder, - "title": folder, + "name": folder, "mimeType": FOLDER_MIME_TYPE, - "parents": [{"id": "root"}], + "parents": ["root"], } for folder in folders ) return iter( { - "title": "data", + "id": f"data-{parent}", + "name": "data", "mimeType": "application/octet-stream", - "parents": [{"id": parent}], - "fileSize": "7", + "parents": [parent], + "size": "7", "md5Checksum": "checksum", } for parent in parents diff --git a/pydrive2/test/test_fs_unit.py b/pydrive2/test/test_fs_unit.py new file mode 100644 index 0000000..a2e1a25 --- /dev/null +++ b/pydrive2/test/test_fs_unit.py @@ -0,0 +1,205 @@ +import json +from urllib.parse import urlsplit + +import pytest +from fsspec.callbacks import Callback + +from pydrive2.fs import GDriveFileSystem +from pydrive2.fs.spec import FOLDER_MIME_TYPE +from pydrive2.test.http_mock import error_response, query, response + + +def filesystem(drive, root="root", **kwargs): + return GDriveFileSystem( + root, google_auth=drive.auth, skip_instance_cache=True, **kwargs + ) + + +@pytest.mark.parametrize("root", ["root", "appDataFolder"]) +def test_find_accepts_canonical_parent_of_root_alias(mock_drive, root): + item = { + "id": "child", + "name": "note", + "mimeType": "text/plain", + "parents": ["canonical-root-id"], + "size": "5", + } + drive, requests = mock_drive(response({"files": [item]})) + assert filesystem(drive, root).find(root) == [f"{root}/note"] + (listing,) = requests + assert f"'{root}' in parents" in query(listing)["q"][0] + if root == "appDataFolder": + assert query(listing)["spaces"] == ["appDataFolder"] + + +def test_ls_deduplicates_a_file_shared_across_parent_batches(mock_drive): + parents = [f"parent-{i}" for i in range(51)] + shared = { + "id": "shared", + "name": "same", + "mimeType": "text/plain", + "parents": [parents[0], parents[-1]], + "size": "1", + } + other = dict(shared, id="distinct", parents=[parents[-1]], size="2") + drive, requests = mock_drive( + response({"files": [shared]}), response({"files": [shared, other]}) + ) + fs = filesystem(drive) + fs._cache_path_id("duplicate", *parents) + assert [ + (item["name"], item["size"]) + for item in fs.ls("root/duplicate", detail=True) + ] == [("root/duplicate/same", 1), ("root/duplicate/same", 2)] + first, second = requests + assert query(first)["q"][0].count(" in parents") == 50 + assert query(second)["q"][0].count(" in parents") == 1 + + +def test_find_matches_legacy_parent_before_assuming_root(mock_drive): + folder = { + "id": "dir", + "name": "sub", + "mimeType": FOLDER_MIME_TYPE, + "parents": ["canonical-root"], + } + file = { + "id": "file", + "name": "child", + "mimeType": "text/plain", + "parents": ["elsewhere", "dir"], + } + drive, _ = mock_drive( + response({"files": [folder]}), response({"files": [file]}) + ) + fs = filesystem(drive) + assert fs.exists("root/sub") + assert fs.find("root") == ["root/sub/child"] + + +@pytest.mark.parametrize("trash_only", [False, True]) +def test_shared_drive_delete_permission_error(mock_drive, trash_only): + file = {"id": "file", "name": "note", "mimeType": "text/plain"} + drive, _ = mock_drive( + response({"driveId": "shared"}), + response({"files": [file]}), + error_response(403, "insufficientFilePermissions"), + ) + with pytest.raises(PermissionError, match="Manager"): + filesystem(drive, "shared", trash_only=trash_only).rm_file( + "shared/note" + ) + + +def test_copy_creates_missing_directories_and_copies_on_drive(mock_drive): + source = {"name": "payload", "id": "source", "mimeType": "text/plain"} + drive, requests = mock_drive( + response({"files": [source]}), + response({"files": []}), + response({"id": "new"}), + response({"files": []}), + response({"id": "nested"}), + response({"id": "copy"}), + ) + filesystem(drive).cp_file("root/payload", "root/new/nested/copy") + ( + source_lookup, + new_lookup, + new_folder, + nested_lookup, + nested_folder, + copy, + ) = requests + assert [r[1] for r in requests] == [ + "GET", + "GET", + "POST", + "GET", + "POST", + "POST", + ] + assert "name='payload'" in query(source_lookup)["q"][0] + assert "'root' in parents" in query(new_lookup)["q"][0] + assert json.loads(new_folder[2])["parents"] == ["root"] + assert "'new' in parents" in query(nested_lookup)["q"][0] + assert json.loads(nested_folder[2])["parents"] == ["new"] + assert urlsplit(copy[0]).path == "/drive/v3/files/source/copy" + assert json.loads(copy[2]) == {"name": "copy", "parents": ["nested"]} + + +@pytest.mark.parametrize( + ("code", "reason"), [(503, "backendError"), (403, "rateLimitExceeded")] +) +def test_copy_retries_only_copy_request(mock_drive, code, reason): + source = {"name": "payload", "id": "source", "mimeType": "text/plain"} + drive, requests = mock_drive( + response({"files": [source]}), + error_response(code, reason), + response({"id": "copy"}), + ) + filesystem(drive).cp_file("root/payload", "root/copy") + lookup, first, retry = requests + assert lookup[1] == "GET" + assert first[:3] == retry[:3] + assert urlsplit(retry[0]).path.endswith("/source/copy") + + +def test_info_and_listing_request_only_required_fields(mock_drive): + item = { + "name": "payload", + "id": "blob", + "mimeType": "text/plain", + "size": "3", + "md5Checksum": "abc", + } + drive, requests = mock_drive(response({"files": [item]}), response(item)) + assert filesystem(drive).info("root/payload") == { + "name": "root/payload", + "type": "file", + "size": 3, + "checksum": "abc", + } + lookup, get = requests + assert query(lookup)["fields"] == [ + "nextPageToken,files(id,name,parents,mimeType,size,md5Checksum)" + ] + assert query(get)["fields"] == ["mimeType,size,md5Checksum"] + + +@pytest.mark.parametrize( + ("media", "expected"), + [ + ( + [ + ({"status": "206", "content-range": "bytes 0-1/3"}, b"xy"), + ({"status": "206", "content-range": "bytes 2-2/3"}, b"z"), + ], + [(3, 0), (3, 2), (3, 3)], + ), + ([({"status": "200"}, b"xyz")], [(None, 3)]), + ], + ids=["two-chunks", "unknown-total"], +) +def test_download_progress_uses_media_responses( + mock_drive, tmp_path, media, expected +): + source = { + "name": "payload", + "id": "file", + "mimeType": "application/octet-stream", + } + drive, requests = mock_drive(response({"files": [source]}), *media) + updates = [] + callback = Callback( + hooks={"record": lambda size, value: updates.append((size, value))} + ) + dest = tmp_path / "result" + filesystem(drive).get_file( + "root/payload", dest, callback=callback, block_size=2 + ) + lookup, *downloads = requests + assert query(lookup)["q"] + assert len(downloads) == len(media) + assert all(query(request)["alt"] == ["media"] for request in downloads) + assert updates == expected + assert dest.read_bytes() == b"xyz" diff --git a/pydrive2/test/test_util.py b/pydrive2/test/test_util.py index e10dc09..70df4cc 100644 --- a/pydrive2/test/test_util.py +++ b/pydrive2/test/test_util.py @@ -58,7 +58,7 @@ def pydrive_retry(call, *args, **kwargs): def pydrive_list_item(drive, query, max_results=1000): - param = {"q": query, "maxResults": max_results} + param = {"q": query, "pageSize": max_results} file_list = drive.ListFile(param)