diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 9946f8da..00000000 --- a/.flake8 +++ /dev/null @@ -1,10 +0,0 @@ -[flake8] -ignore = - # Whitespace before ':' - E203, - # Too many leading '#' for block comment - E266, - # Line break occurred before a binary operator - W503 -max-line-length = 89 -select = B,C,E,F,W,T4,B9 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 8f0af424..4a308825 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -9,11 +9,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v7 with: - python-version: '3.8' + python-version: '3.12' - name: Build run: | diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 65947d33..f15f878a 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -9,6 +9,16 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - - uses: pre-commit/action@v3.0.1 + - uses: actions/checkout@v7 + - uses: astral-sh/ruff-action@v4.1.0 + - run: ruff format --check . + + types: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: '3.11' + - run: python -m pip install -e '.[fsspec,tests,typing]' + - run: python -m mypy diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index baf34ac8..fd4c7793 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -17,11 +17,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v7 with: - python-version: '3.8' + python-version: '3.12' - name: Install pypa/build run: python -m pip install build diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d24ab93e..6e7a4f18 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,6 +3,7 @@ on: branches: - main pull_request_target: + workflow_dispatch: name: Tests @@ -24,7 +25,7 @@ jobs: max-parallel: 3 matrix: os: [ubuntu-latest, macos-latest, windows-latest] - pyv: ["3.9", "3.10", "3.11", "3.12"] + pyv: ["3.11", "3.12", "3.13", "3.14", "3.15"] steps: - uses: actions/checkout@v4 @@ -32,7 +33,7 @@ jobs: # NOTE: needed for pull_request_target to use PR code ref: ${{ github.event.pull_request.head.sha || github.ref }} - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v7 with: python-version: ${{ matrix.pyv }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a85ab87c..01813e6b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,11 +1,7 @@ repos: -- hooks: - - id: black - language_version: python3 - repo: https://github.com/psf/black - rev: 23.12.1 -- hooks: - - id: flake8 - language_version: python3 - repo: https://github.com/PyCQA/flake8 - rev: 6.0.0 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.17.6 + hooks: + - id: ruff-check + - id: ruff-format + types_or: [python, pyi] diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 476d57e6..ba5f5445 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -11,3 +11,7 @@ Contributing code If you have improvements to PyDrive2, send us your pull requests! For those just getting started, Github has a `howto `_. + +Install the development tools with ``python -m pip install -e '.[tests,fsspec,typing]'``. +Run ``ruff check .`` to lint Python files and ``ruff format .`` to format them. +Run ``python -m mypy`` to type-check the package for Python 3.11. diff --git a/README.rst b/README.rst index feacb87b..8d064426 100644 --- a/README.rst +++ b/README.rst @@ -44,7 +44,7 @@ Features of PyDrive2 How to install -------------- -You can install PyDrive2 with regular ``pip`` command. +PyDrive2 requires Python 3.11 or newer. Install it with ``pip``. :: diff --git a/docs/conf.py b/docs/conf.py index df4e8e23..64112a5b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # PyDrive2 documentation build configuration file, created by # sphinx-quickstart on Sun Jun 12 23:01:40 2016. @@ -12,8 +11,8 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys import os +import sys # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the diff --git a/examples/Upload-and-autoconvert-to-Google-Drive-Format-Example/upload.py b/examples/Upload-and-autoconvert-to-Google-Drive-Format-Example/upload.py index 05b6642c..5704a125 100644 --- a/examples/Upload-and-autoconvert-to-Google-Drive-Format-Example/upload.py +++ b/examples/Upload-and-autoconvert-to-Google-Drive-Format-Example/upload.py @@ -7,8 +7,10 @@ usage: upload.py example usage: upload.py 0B5XXXXY9KddXXXXXXXA2c3ZXXXX /path/to/my/file """ + import sys from os import path + from pydrive2.auth import GoogleAuth from pydrive2.drive import GoogleDrive from pydrive2.settings import LoadSettingsFile @@ -24,9 +26,9 @@ drive = GoogleDrive(gauth) # If provided arguments incorrect, print usage instructions and exit. -if len(sys.argv) < 2: +if len(sys.argv) < 3: print("usage: upload.py ") - exit(1) # Exit program as incorrect parameters provided. + sys.exit(1) parentId = sys.argv[1] myFilePath = sys.argv[2] @@ -35,9 +37,7 @@ # Check if file name already exists in folder. file_list = drive.ListFile( { - "q": '"{}" in parents and title="{}" and trashed=false'.format( - parentId, myFileName - ) + "q": f'"{parentId}" in parents and title="{myFileName}" and trashed=false' } ).GetList() diff --git a/examples/strip_bom_example.py b/examples/strip_bom_example.py index 5b44b844..b595bd68 100644 --- a/examples/strip_bom_example.py +++ b/examples/strip_bom_example.py @@ -20,8 +20,8 @@ # Print content before download. print("Original text:") print(bytes(original_file_content.encode("unicode-escape"))) -print("Number of chars: %d" % len(original_file_content)) -print("") +print(f"Number of chars: {len(original_file_content)}") +print() # Original text: # Generic, non-exhaustive\n ASCII test string. # Number of chars: 43 @@ -31,8 +31,8 @@ content_with_bom = file2.GetContentString(mimetype="text/plain") print("Content with BOM:") print(bytes(content_with_bom.encode("unicode-escape"))) -print("Number of chars: %d" % len(content_with_bom)) -print("") +print(f"Number of chars: {len(content_with_bom)}") +print() # Content with BOM: # \ufeffGeneric, non-exhaustive\r\n ASCII test string. # Number of chars: 45 @@ -44,8 +44,8 @@ ) print("Content without BOM:") print(bytes(content_without_bom.encode("unicode-escape"))) -print("Number of chars: %d" % len(content_without_bom)) -print("") +print(f"Number of chars: {len(content_without_bom)}") +print() # Content without BOM: # Generic, non-exhaustive\r\n ASCII test string. # Number of chars: 44 diff --git a/examples/using_folders.py b/examples/using_folders.py index 02d0b85f..58448ed4 100644 --- a/examples/using_folders.py +++ b/examples/using_folders.py @@ -18,7 +18,7 @@ # Get folder info and print to screen. folder_title = folder["title"] folder_id = folder["id"] -print("title: %s, id: %s" % (folder_title, folder_id)) +print(f"title: {folder_title}, id: {folder_id}") # Upload file to folder. f = drive.CreateFile( diff --git a/pydrive2/__pyinstaller/hook-googleapiclient.py b/pydrive2/__pyinstaller/hook-googleapiclient.py index c16f270f..1611f638 100644 --- a/pydrive2/__pyinstaller/hook-googleapiclient.py +++ b/pydrive2/__pyinstaller/hook-googleapiclient.py @@ -1,6 +1,6 @@ from PyInstaller.utils.hooks import ( # pylint: disable=import-error - copy_metadata, collect_data_files, + copy_metadata, ) datas = copy_metadata("google-api-python-client") diff --git a/pydrive2/__pyinstaller/test_hook-googleapiclient.py b/pydrive2/__pyinstaller/test_hook-googleapiclient.py index 811661b2..71f16fa7 100644 --- a/pydrive2/__pyinstaller/test_hook-googleapiclient.py +++ b/pydrive2/__pyinstaller/test_hook-googleapiclient.py @@ -2,25 +2,14 @@ from PyInstaller import __main__ as pyi_main - -# NOTE: importlib.resources.contents is available in py3.7+, but due to how -# pyinstaller handles importlib, we need to use the importlib_resources -# backport if there are any resources methods that are not available in a given -# python version, which ends up being py<3.10 _APP_SOURCE = """ -import sys -if sys.version_info >= (3, 10): - from importlib.resources import contents -else: - from importlib_resources import contents +from importlib.resources import files import pydrive2.files -cache_files = contents( - "googleapiclient.discovery_cache.documents" -) -assert len(cache_files) > 0 +documents = files("googleapiclient.discovery_cache.documents") +assert documents.joinpath("drive.v2.json").is_file() """ diff --git a/pydrive2/apiattr.py b/pydrive2/apiattr.py index ef84645f..d0254340 100644 --- a/pydrive2/apiattr.py +++ b/pydrive2/apiattr.py @@ -99,9 +99,10 @@ def GetChanges(self): """ dirty = {} for key in self: - if self.metadata.get(key) is None: - dirty[key] = self[key] - elif self.metadata[key] != self[key]: + if ( + self.metadata.get(key) is None + or self.metadata[key] != self[key] + ): dirty[key] = self[key] return dirty diff --git a/pydrive2/auth.py b/pydrive2/auth.py index b188adf1..ce2b1850 100644 --- a/pydrive2/auth.py +++ b/pydrive2/auth.py @@ -1,28 +1,32 @@ -import os import json -import webbrowser -import httplib2 -import oauth2client.clientsecrets as clientsecrets +import os import threading +import webbrowser +from functools import wraps +from typing import ClassVar +import httplib2 from googleapiclient.discovery import build -from functools import wraps -from oauth2client.service_account import ServiceAccountCredentials -from oauth2client.client import FlowExchangeError -from oauth2client.client import AccessTokenRefreshError -from oauth2client.client import OAuth2WebServerFlow -from oauth2client.client import OOB_CALLBACK_URN +from oauth2client import clientsecrets +from oauth2client._helpers import scopes_to_string +from oauth2client.client import ( + OOB_CALLBACK_URN, + AccessTokenRefreshError, + FlowExchangeError, + OAuth2WebServerFlow, +) from oauth2client.contrib.dictionary_storage import DictionaryStorage from oauth2client.file import Storage -from oauth2client.tools import ClientRedirectHandler -from oauth2client.tools import ClientRedirectServer -from oauth2client._helpers import scopes_to_string -from .apiattr import ApiAttribute -from .apiattr import ApiAttributeMixin -from .settings import LoadSettingsFile -from .settings import ValidateSettings -from .settings import SettingsError -from .settings import InvalidConfigError +from oauth2client.service_account import ServiceAccountCredentials +from oauth2client.tools import ClientRedirectHandler, ClientRedirectServer + +from .apiattr import ApiAttribute, ApiAttributeMixin +from .settings import ( + InvalidConfigError, + LoadSettingsFile, + SettingsError, + ValidateSettings, +) class AuthError(Exception): @@ -150,13 +154,13 @@ class GoogleAuth(ApiAttributeMixin): and authorization. """ - DEFAULT_SETTINGS = { + DEFAULT_SETTINGS: ClassVar[dict[str, str | bool | list[str]]] = { "client_config_backend": "file", "client_config_file": "client_secrets.json", "save_credentials": False, "oauth_scope": ["https://www.googleapis.com/auth/drive"], } - CLIENT_CONFIGS_LIST = [ + CLIENT_CONFIGS_LIST: ClassVar[list[str]] = [ "client_id", "client_secret", "auth_uri", @@ -164,7 +168,7 @@ class GoogleAuth(ApiAttributeMixin): "revoke_uri", "redirect_uri", ] - SERVICE_CONFIGS_LIST = ["client_user_email"] + SERVICE_CONFIGS_LIST: ClassVar[list[str]] = ["client_user_email"] settings = ApiAttribute("settings") client_config = ApiAttribute("client_config") flow = ApiAttribute("flow") @@ -381,7 +385,7 @@ def _InitializeStoragesFromSettings(self): result[backend] = DictionaryStorage(creds_dict, creds_key) elif save_credentials: raise InvalidConfigError( - "Unknown save_credentials_backend: %s" % backend + f"Unknown save_credentials_backend: {backend}" ) return result, result.get(backend) @@ -549,7 +553,7 @@ def LoadClientConfigFile(self, client_config_file=None): client_config_file ) except clientsecrets.InvalidClientSecretsError as error: - raise InvalidConfigError("Invalid client secrets file %s" % error) + raise InvalidConfigError(f"Invalid client secrets file {error}") if client_type not in ( clientsecrets.TYPE_WEB, clientsecrets.TYPE_INSTALLED, @@ -681,7 +685,7 @@ def Refresh(self): try: self.credentials.refresh(self.http) except AccessTokenRefreshError as error: - raise RefreshError("Access token refresh failed: %s" % error) + raise RefreshError(f"Access token refresh failed: {error}") def GetAuthUrl(self): """Creates authentication url where user visits to grant access. @@ -714,7 +718,7 @@ def Authenticate(self, code): try: self.credentials = self.flow.step2_exchange(code) except FlowExchangeError as e: - raise AuthenticationError("OAuth2 code exchange failed: %s" % e) + raise AuthenticationError(f"OAuth2 code exchange failed: {e}") print("Authentication successful.") def _build_http(self): diff --git a/pydrive2/drive.py b/pydrive2/drive.py index 8189ab14..9a272247 100644 --- a/pydrive2/drive.py +++ b/pydrive2/drive.py @@ -1,7 +1,6 @@ from .apiattr import ApiAttributeMixin -from .files import GoogleDriveFile -from .files import GoogleDriveFileList from .auth import LoadAuth +from .files import GoogleDriveFile, GoogleDriveFileList class GoogleDrive(ApiAttributeMixin): diff --git a/pydrive2/files.py b/pydrive2/files.py index f928e6ce..a611719f 100644 --- a/pydrive2/files.py +++ b/pydrive2/files.py @@ -1,18 +1,22 @@ import io -import os -import mimetypes import json - -from googleapiclient import errors -from googleapiclient.http import MediaIoBaseUpload -from googleapiclient.http import MediaIoBaseDownload -from googleapiclient.http import DEFAULT_CHUNK_SIZE +import mimetypes +import os from functools import wraps -from .apiattr import ApiAttribute -from .apiattr import ApiAttributeMixin -from .apiattr import ApiResource -from .apiattr import ApiResourceList +from googleapiclient import errors +from googleapiclient.http import ( + DEFAULT_CHUNK_SIZE, + MediaIoBaseDownload, + MediaIoBaseUpload, +) + +from .apiattr import ( + ApiAttribute, + ApiAttributeMixin, + ApiResource, + ApiResourceList, +) from .auth import LoadAuth BLOCK_SIZE = 1024 @@ -141,7 +145,7 @@ def __init__( if pre_buffer: self.read() if remove_prefix: - chunk = io.BytesIO(self._fd.chunk) + chunk = io.BytesIO(self._fd.chunk or b"") GoogleDriveFile._RemovePrefix(chunk, remove_prefix) self._fd.chunk = chunk.getvalue() self._pre_buffer = True @@ -259,7 +263,7 @@ def SetContentFile(self, filename): :param filename: name of the file to be uploaded. :type filename: str. """ - self.content = open(filename, "rb") + self.content = open(filename, "rb") # noqa: SIM115 - retained for upload if self.get("id") is None: if self.get("title") is None: self["title"] = os.path.basename(filename) @@ -416,7 +420,7 @@ def GetContentIOBuffer( exc.error["code"] != 403 or exc.GetField("reason") != "fileNotDownloadable" ): - raise exc + raise mimetype = mimetype or "text/plain" request = self._WrapRequest( files.export_media(fileId=file_id, mimeType=mimetype) @@ -699,9 +703,9 @@ def _AddResourceKeyHeaders(self, request): if file_id: resourceKey = self.get("resourceKey") if resourceKey: - request.headers[ - "X-Goog-Drive-Resource-Keys" - ] = f"{file_id}/{resourceKey}" + request.headers["X-Goog-Drive-Resource-Keys"] = ( + f"{file_id}/{resourceKey}" + ) return request @LoadAuth @@ -884,6 +888,7 @@ def _DownloadFromUrl(self, url): :returns: str -- content of downloaded file in string. :raises: ApiRequestError """ + assert self.http is not None # Set by LoadAuth. resp, content = self.http.request(url) if resp.status != 200: raise ApiRequestError(errors.HttpError(resp, content, uri=url)) @@ -908,13 +913,11 @@ def _DeletePermission(self, permission_id): raise ApiRequestError(error) else: if "permissions" in self and "permissions" in self.metadata: - permissions = self["permissions"] - is_not_current_permission = ( - lambda per: per["id"] == permission_id - ) - permissions = list( - filter(is_not_current_permission, permissions) - ) + permissions = [ + permission + for permission in self["permissions"] + if permission["id"] == permission_id + ] self["permissions"] = permissions self.metadata["permissions"] = permissions return True diff --git a/pydrive2/fs/spec.py b/pydrive2/fs/spec.py index b39306eb..6a2b062f 100644 --- a/pydrive2/fs/spec.py +++ b/pydrive2/fs/spec.py @@ -1,4 +1,3 @@ -import appdirs import errno import io import logging @@ -8,14 +7,16 @@ from collections import defaultdict from contextlib import contextmanager from itertools import chain +from typing import TypedDict +import appdirs from fsspec.spec import AbstractFileSystem from funcy import cached_property, retry, wrap_prop, wrap_with from tqdm.utils import CallbackIOWrapper +from pydrive2.auth import GoogleAuth from pydrive2.drive import GoogleDrive from pydrive2.fs.utils import IterStream -from pydrive2.auth import GoogleAuth logger = logging.getLogger(__name__) @@ -30,6 +31,11 @@ } +class _IdsCache(TypedDict): + dirs: defaultdict[str, list[str]] + ids: dict[str, str] + + class GDriveAuthError(Exception): pass @@ -244,8 +250,8 @@ def split_path(self, path): @wrap_prop(threading.RLock()) @cached_property - def _ids_cache(self): - cache = {"dirs": defaultdict(list), "ids": {}} + def _ids_cache(self) -> _IdsCache: + cache: _IdsCache = {"dirs": defaultdict(list), "ids": {}} base_item_ids = self._path_to_item_ids(self.base, use_cache=False) if not base_item_ids: @@ -272,9 +278,7 @@ def _list_params(self): drive_id = self._gdrive_shared_drive_id(self.root) if drive_id: logger.debug( - "GDrive remote '{}' is using shared drive id '{}'.".format( - self.path, drive_id - ) + f"GDrive remote '{self.path}' is using shared drive id '{drive_id}'." ) params["driveId"] = drive_id params["corpora"] = "drive" @@ -471,7 +475,7 @@ def find(self, path, detail=False, **kwargs): self._path_to_item_ids(base) dir_ids = [self._ids_cache["ids"].copy()] - seen_paths = set() + seen_paths: set[str] = set() contents = [] while dir_ids: query_ids = { @@ -485,7 +489,7 @@ def find(self, path, detail=False, **kwargs): seen_paths |= query_ids.keys() - new_query_ids = {} + new_query_ids: dict[str, str] = {} dir_ids.append(new_query_ids) for item in self._gdrive_list_ids(query_ids): parent_id = item["parents"][0]["id"] @@ -680,7 +684,7 @@ def flush(self): def close(self): if self._closed: - return None + return self.flush() self.buffer.close() diff --git a/pydrive2/settings.py b/pydrive2/settings.py index a7d12304..1b4dc112 100644 --- a/pydrive2/settings.py +++ b/pydrive2/settings.py @@ -1,10 +1,7 @@ -from yaml import load -from yaml import YAMLError +import yaml +from yaml import YAMLError, load -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader +SafeLoader = getattr(yaml, "CSafeLoader", yaml.SafeLoader) SETTINGS_FILE = "settings.yaml" SETTINGS_STRUCT = { @@ -163,7 +160,7 @@ def _ValidateSettingsElement(data, struct, key): try: default = struct[key]["default"] except KeyError: - raise InvalidConfigError("Missing required setting %s" % key) + raise InvalidConfigError(f"Missing required setting {key}") else: data[key] = default # If data exists, Check type of the data @@ -177,8 +174,7 @@ def _ValidateSettingsElement(data, struct, key): for element in data[key]: if not isinstance(element, struct[key]["struct"]): raise InvalidConfigError( - "Setting %s should be list of %s" - % (key, struct[key]["struct"]) + f"Setting {key} should be list of {struct[key]['struct']}" ) # Check dependency of this attribute. dependencies = struct[key].get("dependency") diff --git a/pydrive2/test/test_apiattr.py b/pydrive2/test/test_apiattr.py index a1cbd8a1..59c081a5 100644 --- a/pydrive2/test/test_apiattr.py +++ b/pydrive2/test/test_apiattr.py @@ -4,8 +4,8 @@ from pydrive2.drive import GoogleDrive from pydrive2.test.test_util import ( pydrive_retry, - setup_credentials, settings_file_path, + setup_credentials, ) diff --git a/pydrive2/test/test_drive.py b/pydrive2/test/test_drive.py index 81498a80..c69d4df6 100644 --- a/pydrive2/test/test_drive.py +++ b/pydrive2/test/test_drive.py @@ -4,8 +4,8 @@ from pydrive2.drive import GoogleDrive from pydrive2.test.test_util import ( pydrive_retry, - setup_credentials, settings_file_path, + setup_credentials, ) diff --git a/pydrive2/test/test_file.py b/pydrive2/test/test_file.py index 573ff322..acde29a2 100644 --- a/pydrive2/test/test_file.py +++ b/pydrive2/test/test_file.py @@ -1,17 +1,18 @@ import filecmp import json import os -import unittest -from unittest.mock import MagicMock -import httplib2 -import pytest import sys +import unittest +from concurrent.futures import ThreadPoolExecutor, as_completed from io import BytesIO +from pathlib import Path from tempfile import mkdtemp from time import time +from unittest.mock import MagicMock +import httplib2 +import pytest import timeout_decorator -from concurrent.futures import ThreadPoolExecutor, as_completed from googleapiclient import errors from pydrive2.auth import GoogleAuth @@ -19,12 +20,12 @@ from pydrive2.files import ApiRequestError, GoogleDriveFile from pydrive2.test import test_util from pydrive2.test.test_util import ( - pydrive_retry, - setup_credentials, create_file, delete_dir, delete_file, + pydrive_retry, settings_file_path, + setup_credentials, ) @@ -829,7 +830,7 @@ def test_GFile_Conversion_Lossless_String(self): mimetype="text/plain", remove_bom=True, ) - downloaded_string = open(downloaded_file_name).read() + downloaded_string = Path(downloaded_file_name).read_text() self.assertEqual( test_string, downloaded_string, "Strings do not match" ) @@ -848,9 +849,7 @@ def setup_gfile_conversion_test(self): file_name = "_tmp_source_file.txt" downloaded_file_name = "_tmp_downloaded_file_name.txt" original_file_content = "Generic, non-exhaustive\n ASCII test string." - source_file = open(file_name, mode="w+") - source_file.write(original_file_content) - source_file.close() + Path(file_name).write_text(original_file_content) original_file_content = test_util.StripNewlines(original_file_content) return file1, file_name, original_file_content, downloaded_file_name @@ -890,7 +889,7 @@ def test_GFile_Conversion_Remove_BOM(self): pydrive_retry( file1.GetContentFile, downloaded_file_name, remove_bom=True ) - downloaded_content = open(downloaded_file_name).read() + downloaded_content = Path(downloaded_file_name).read_text() downloaded_content = test_util.StripNewlines(downloaded_content) self.assertEqual(original_file_content, downloaded_content) @@ -905,7 +904,7 @@ def test_Gfile_Conversion_Add_Remove_BOM(self): ( file1, file_name, - original_file_content, + _, downloaded_file_name, ) = self.setup_gfile_conversion_test() try: @@ -1021,7 +1020,7 @@ def _parallel_uploader(self, num_of_uploads, num_of_workers): # Ensure there are no files with the random file name. files = pydrive_retry( lambda: drive.ListFile( - param={"q": "title = '%s' and trashed = false" % remote_name} + param={"q": f"title = '{remote_name}' and trashed = false"} ).GetList() ) self.assertTrue(len(files) == 0) @@ -1036,15 +1035,22 @@ def _parallel_uploader(self, num_of_uploads, num_of_workers): self.assertIsNone(future.exception()) thread_pool.shutdown() - # Ensure all files were uploaded. - files = pydrive_retry( - lambda: drive.ListFile( - param={"q": "title = '%s' and trashed = false" % remote_name} + file_ids = [file["id"] for file in upload_files] + self.assertEqual(len(set(file_ids)), num_of_uploads) + + def check_uploaded_files(): + # A successful upload can take time to appear in title searches. + files = drive.ListFile( + param={"q": f"title = '{remote_name}' and trashed = false"} ).GetList() - ) - self.assertTrue(len(files) == self.FILE_UPLOAD_COUNT) + if len(files) < num_of_uploads: + raise test_util.PyDriveRetriableError( + f"Drive lists {len(files)} of {num_of_uploads} uploads" + ) + self.assertCountEqual([file["id"] for file in files], file_ids) - return [fi["id"] for fi in upload_files] + pydrive_retry(check_uploaded_files) + return file_ids def _parallel_downloader(self, file_ids, num_of_workers): drive = GoogleDrive(self.ga) diff --git a/pydrive2/test/test_filelist.py b/pydrive2/test/test_filelist.py index 15918cda..37319274 100644 --- a/pydrive2/test/test_filelist.py +++ b/pydrive2/test/test_filelist.py @@ -5,10 +5,10 @@ from pydrive2.drive import GoogleDrive from pydrive2.test import test_util from pydrive2.test.test_util import ( - pydrive_retry, pydrive_list_item, - setup_credentials, + pydrive_retry, settings_file_path, + setup_credentials, ) @@ -38,11 +38,7 @@ def test_01_Files_List_GetList(self): def test_02_Files_List_ForLoop(self): drive = GoogleDrive(self.ga) query = f"title = '{self.title}' and trashed = false" - files = [] - for x in pydrive_list_item( - drive, query, 2 - ): # Build iterator to access files simply with for loop - files.append(x) + files = list(pydrive_list_item(drive, query, 2)) for file1 in self.file_list: found = False for file2 in files: @@ -54,7 +50,7 @@ def test_03_Files_List_GetList_Iterate(self): drive = GoogleDrive(self.ga) flist = drive.ListFile( { - "q": "title = '%s' and trashed = false" % self.title, + "q": f"title = '{self.title}' and trashed = false", "maxResults": 2, } ) @@ -96,7 +92,7 @@ def test_File_List_Folders(self): def setUp(self): title = test_util.CreateRandomFileName() file_list = [] - for x in range(0, 10): + for x in range(10): file1 = self.drive.CreateFile() file1["title"] = title pydrive_retry(file1.Upload) diff --git a/pydrive2/test/test_fs.py b/pydrive2/test/test_fs.py index 483ab7eb..15878340 100644 --- a/pydrive2/test/test_fs.py +++ b/pydrive2/test/test_fs.py @@ -1,16 +1,20 @@ -from io import StringIO import os import posixpath import secrets import uuid from concurrent import futures +from io import StringIO -import pytest import fsspec +import pytest + from pydrive2.auth import GoogleAuth from pydrive2.fs import GDriveFileSystem -from pydrive2.test.test_util import settings_file_path, setup_credentials -from pydrive2.test.test_util import GDRIVE_USER_CREDENTIALS_DATA +from pydrive2.test.test_util import ( + GDRIVE_USER_CREDENTIALS_DATA, + settings_file_path, + setup_credentials, +) TEST_GDRIVE_REPO_BUCKET = "root" @@ -65,7 +69,7 @@ def fs(fs_factory): def test_fs_oauth(base_remote_dir): GDriveFileSystem( base_remote_dir, - client_id="47794215776-cd9ssb6a4vv5otkq6n0iadpgc4efgjb1.apps.googleusercontent.com", # noqa: E501 + client_id="47794215776-cd9ssb6a4vv5otkq6n0iadpgc4efgjb1.apps.googleusercontent.com", client_secret="i2gerGA7uBjZbR08HqSOSt9Z", ) @@ -278,13 +282,9 @@ def test_find(fs, fs_factory, remote_dir): for test_fs in [fs, fs_factory()[0]]: # Test for https://github.com/iterative/PyDrive2/issues/229 # It must go first, so that we test with a cache miss as well - assert set(test_fs.find(remote_dir + "/dir/c/d/")) == set( - [ - file - for file in files - if file.startswith(remote_dir + "/dir/c/d/") - ] - ) + assert set(test_fs.find(remote_dir + "/dir/c/d/")) == { + file for file in files if file.startswith(remote_dir + "/dir/c/d/") + } # General find test assert set(test_fs.find(remote_dir)) == set(files) @@ -296,9 +296,8 @@ def test_find(fs, fs_factory, remote_dir): def test_exceptions(fs, tmpdir, remote_dir): - with pytest.raises(FileNotFoundError): - with fs.open(remote_dir + "/a.txt"): - ... + with pytest.raises(FileNotFoundError), fs.open(remote_dir + "/a.txt"): + ... with pytest.raises(FileNotFoundError): fs.copy(remote_dir + "/u.txt", remote_dir + "/y.txt") diff --git a/pydrive2/test/test_oauth.py b/pydrive2/test/test_oauth.py index 5c29e18c..4ab0078f 100644 --- a/pydrive2/test/test_oauth.py +++ b/pydrive2/test/test_oauth.py @@ -2,16 +2,17 @@ import os import re import time + import pytest +from oauth2client.file import Storage from pydrive2.auth import AuthenticationError, GoogleAuth from pydrive2.test.test_util import ( - setup_credentials, + GDRIVE_USER_CREDENTIALS_DATA, delete_file, settings_file_path, - GDRIVE_USER_CREDENTIALS_DATA, + setup_credentials, ) -from oauth2client.file import Storage def setup_module(module): diff --git a/pydrive2/test/test_util.py b/pydrive2/test/test_util.py index aeff678e..e10dc09a 100644 --- a/pydrive2/test/test_util.py +++ b/pydrive2/test/test_util.py @@ -1,13 +1,14 @@ -from pathlib import Path -import random -import re import os import posixpath +import random +import re +from itertools import chain +from pathlib import Path +from shutil import copyfile, rmtree from funcy import retry -from funcy.py3 import cat + from pydrive2.files import ApiRequestError -from shutil import copyfile, rmtree newline_pattern = re.compile(r"[\r\n]") @@ -62,15 +63,14 @@ def pydrive_list_item(drive, query, max_results=1000): file_list = drive.ListFile(param) # Isolate and decorate fetching of remote drive items in pages - get_list = lambda: pydrive_retry(next, file_list, None) # noqa: E731 + get_list = lambda: pydrive_retry(next, file_list, None) # Fetch pages until None is received, lazily flatten the thing - return cat(iter(get_list, None)) + return chain.from_iterable(iter(get_list, None)) def CreateRandomFileName(): - hash = random.getrandbits(128) - return "%032x" % hash + return f"{random.getrandbits(128):032x}" def StripNewlines(string): diff --git a/pyproject.toml b/pyproject.toml index 881c8983..20623ba7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,12 +15,13 @@ maintainers = [ classifiers = [ "Development Status :: 4 - Beta", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", ] -requires-python = ">=3.8" +requires-python = ">=3.11" dependencies = [ "google-api-python-client>=1.12.5", "oauth2client>=4.0.0", @@ -48,13 +49,11 @@ tests = [ "pytest>=4.6.0", "timeout-decorator", "funcy>=1.14", - "flake8", - "flake8-docstrings", "pytest-mock", "pyinstaller", - "importlib_resources<6; python_version < '3.10'", - "black==24.10.0", + "ruff==0.17.6", ] +typing = ["mypy==2.4.1", "types-httplib2", "types-PyYAML"] [project.entry-points.pyinstaller40] hook-dirs = "pydrive2.__pyinstaller:get_hook_dirs" @@ -70,23 +69,26 @@ packages=[ [tool.setuptools_scm] -[tool.black] +[tool.ruff] line-length = 79 -include = '\.pyi?$' -exclude = ''' -/( - \.eggs - | \.git - | \.hg - | \.mypy_cache - | \.tox - | \.venv - | _build - | buck-out - | build - | dist -)/ -''' + +[tool.ruff.lint] +extend-select = ["E501", "I", "TC", "UP"] + +[tool.ruff.lint.per-file-ignores] +# PyInstaller discovers hook modules by their hyphenated filenames. +"pydrive2/__pyinstaller/*-*.py" = ["N999"] + +[tool.ruff.lint.pycodestyle] +max-line-length = 89 + +[tool.mypy] +python_version = "3.11" +files = ["pydrive2"] +exclude = ["/test(/|_)"] +check_untyped_defs = true +# Some runtime dependencies do not distribute type information. +disable_error_code = ["import-untyped"] [tool.pytest.ini_options] markers = [