From 2fd671d9815345d96a05f6de98bd214bfbc792c3 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Thu, 19 Nov 2026 15:29:17 -0800 Subject: [PATCH 01/15] Migrate Google Drive authentication to google-auth --- docs/oauth.rst | 30 +- pydrive2/_credentials.py | 188 +++++++++++ pydrive2/auth.py | 458 +++++++++++---------------- pydrive2/settings.py | 2 +- pydrive2/test/test_auth_migration.py | 335 ++++++++++++++++++++ pydrive2/test/test_oauth.py | 4 +- pyproject.toml | 10 +- 7 files changed, 726 insertions(+), 301 deletions(-) create mode 100644 pydrive2/_credentials.py create mode 100644 pydrive2/test/test_auth_migration.py diff --git a/docs/oauth.rst b/docs/oauth.rst index 493a622..091e43a 100644 --- a/docs/oauth.rst +++ b/docs/oauth.rst @@ -15,26 +15,20 @@ OAuth2.0 is complex and difficult to start with. To make it more simple, # Create local webserver and auto handles authentication. gauth.LocalWebserverAuth() - # Or use the CommandLineAuth(), which provides you with a link to paste - # into your browser. The site it leads to then provides you with an - # authentication token which you paste into the command line. - # Commented out as it is an alternative to the LocalWebserverAuth() above, - # and someone will just copy-paste the entire thing into their editor. - - # gauth.CommandLineAuth() - To make this code work, you need to download the application configurations file from APIs Console. Take a look at quickstart_ for detailed instructions. `LocalWebserverAuth()`_ is a built-in method of `GoogleAuth`_ which sets up local webserver to automatically receive authentication code from user and -authorizes by itself. You can also use `CommandLineAuth()`_ which manually -takes code from user at command line. +authorizes by itself. `CommandLineAuth()`_ prints the login URL and waits for +the same local callback. Open the URL in a browser that can reach the +callback port on the machine running PyDrive2. For unattended jobs, use a +service account or previously saved user credentials with a refresh token. .. _quickstart: /PyDrive2/quickstart/#authentication .. _`LocalWebserverAuth()`: /PyDrive2/pydrive2/#pydrive2.auth.GoogleAuth.LocalWebserverAuth .. _`GoogleAuth`: /PyDrive2/pydrive2/#pydrive2.auth.GoogleAuth -.. _`CommandLineAuth()`: /PyDrive2/pydrive2/#pydrive.auth.GoogleAuth.CommandLineAuth +.. _`CommandLineAuth()`: /PyDrive2/pydrive2/#pydrive2.auth.GoogleAuth.CommandLineAuth Automatic and custom authentication with *settings.yaml* -------------------------------------------------------- @@ -96,7 +90,7 @@ Fields explained: :client_config['client_secret'] (str): Client secret of the application. **Required**: Yes, only if *client_config_backend* is 'settings' :client_config['auth_uri'] (str): The authorization server endpoint URI. **Default**: 'https://accounts.google.com/o/oauth2/auth'. **Required**: No. :client_config['token_uri'] (str): The token server endpoint URI. **Default**: 'https://accounts.google.com/o/oauth2/token'. **Required**: No. -:client_config['redirect_uri'] (str): Redirection endpoint URI. **Default**: 'urn:ietf:wg:oauth:2.0:oob'. **Required**: No. +:client_config['redirect_uri'] (str): Redirection endpoint URI. **Default**: 'http://localhost'. **Required**: No. :client_config['revoke_uri'] (str): Revoke endpoint URI. **Default**: None. **Required**: No. :service_config (dict): Place holding dictionary for client configuration when *client_config_backend* is 'service' or 'settings' and using service account. **Required**: Yes, only if *client_config_backend* is 'service' or 'settings' and not using *client_config* :service_config['client_user_email'] (str): User email that authority was delegated_ to. **Required**: No. @@ -113,6 +107,12 @@ Fields explained: .. _delegated: https://developers.google.com/admin-sdk/directory/v1/guides/delegation +PyDrive2 reads existing credentials saved by ``oauth2client`` from either +backend. After a refresh or explicit save, it writes the new format. Back up +the cache first if an older PyDrive2 installation also uses it. An explicitly +saved service-account credential includes the signing key so it can refresh +independently of the original key file. + Sample *settings.yaml* ______________________ @@ -142,7 +142,7 @@ want to integrate your existing website with Drive API. In that case, you can customize an authentication flow as follows: 1. Get authentication Url from `GetAuthUrl()`_. -2. Ask users to visit the authentication Url and grant access to your application. Retrieve authentication code manually by user or automatically by building your own oauth2callback. +2. Direct the user to the authorization URL. Receive the authorization code at the redirect URI registered for your application and verify the OAuth state in the callback. 3. Call `Auth(code)`_ with the authentication code you retrieved from step 2. Your *settings.yaml* will work for your customized authentication flow, too. @@ -153,9 +153,9 @@ Here is a sample code for your customized authentication flow from pydrive2.auth import GoogleAuth - gauth = GoogleAuth() + gauth = GoogleAuth() # Configure your registered redirect URI in settings.yaml auth_url = gauth.GetAuthUrl() # Create authentication url user needs to visit - code = AskUserToVisitLinkAndGiveCode(auth_url) # Your customized authentication flow + code = ReceiveVerifiedCallback(auth_url) # Your application handles the redirect gauth.Auth(code) # Authorize and build service from the code .. _`GetAuthUrl()`: /PyDrive2/pydrive2/#pydrive2.auth.GoogleAuth.GetAuthUrl diff --git a/pydrive2/_credentials.py b/pydrive2/_credentials.py new file mode 100644 index 0000000..7219b3e --- /dev/null +++ b/pydrive2/_credentials.py @@ -0,0 +1,188 @@ +"""Credential persistence for GoogleAuth's file and dictionary backends.""" + +import base64 +import json +import os +import tempfile +import threading +from datetime import datetime +from pathlib import Path + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.serialization import pkcs12 +from google.oauth2 import credentials, service_account + + +class InvalidCredentialsError(IOError): + """Error trying to read or write credentials.""" + + +class _PersistentCredentials: + storage = None + + def refresh(self, request): + storage = self.storage + if storage is None: + return super().refresh(request) + + # AuthorizedHttp can refresh during a request, including after a 401. + # Persist rotated tokens on that path as well as on explicit Refresh(). + previous_token = self.token + with storage.lock: + if self.token == previous_token or not self.valid: + super().refresh(request) + storage.put(self) + + +class UserCredentials(_PersistentCredentials, credentials.Credentials): + pass + + +class ServiceCredentials(_PersistentCredentials, service_account.Credentials): + @classmethod + def from_key_info(cls, info, scopes, subject): + info = dict(info) + info.setdefault("token_uri", "https://oauth2.googleapis.com/token") + cred = cls.from_service_account_info( + info, scopes=scopes, subject=subject + ) + # An explicitly saved service credential must be able to sign again + # even if the original key file is no longer present. + cred._saved_key_info = info + cred._saved_subject = subject + return cred + + def to_json(self): + return json.dumps( + { + "type": "service_account", + "key": self._saved_key_info, + "scopes": self.scopes, + "subject": self._saved_subject, + "token": self.token, + "expiry": ( + self.expiry.isoformat() + "Z" if self.expiry else None + ), + } + ) + + +def p12_key_info(data, email, password=b"notasecret"): + key, _, _ = pkcs12.load_key_and_certificates(data, password) + if key is None: + raise InvalidCredentialsError("No private key in PKCS12 file") + pem = key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ) + return { + "client_email": email, + "private_key": pem.decode("ascii"), + "token_uri": "https://oauth2.googleapis.com/token", + } + + +def load_credentials(serialized, storage=None): + if not serialized: + return None + try: + data = json.loads(serialized) + scopes = data.get("scopes") + if data.get("_class") == "ServiceAccountCredentials": + # Migrate credentials previously written by oauth2client. + email = data["_service_account_email"] + if data.get("_private_key_pkcs12"): + info = p12_key_info( + base64.b64decode(data["_private_key_pkcs12"]), + email, + ( + data.get("_private_key_password") or "notasecret" + ).encode(), + ) + else: + info = { + "client_email": email, + "private_key": data["_private_key_pkcs8_pem"], + "private_key_id": data.get("_private_key_id"), + } + info["token_uri"] = data["token_uri"] + cred = ServiceCredentials.from_key_info( + info, data["_scopes"].split(), data["_kwargs"].get("sub") + ) + elif data.get("type") == "service_account": + cred = ServiceCredentials.from_key_info( + data["key"], scopes, data.get("subject") + ) + else: + cred = UserCredentials( + token=None, + refresh_token=data.get("refresh_token"), + token_uri=data.get( + "token_uri", "https://oauth2.googleapis.com/token" + ), + client_id=data["client_id"], + client_secret=data["client_secret"], + scopes=scopes, + rapt_token=data.get("rapt_token"), + quota_project_id=data.get("quota_project_id"), + ) + if not data.get("invalid"): + cred.token = data.get("token", data.get("access_token")) + expiry = data.get("expiry", data.get("token_expiry")) + if expiry: + cred.expiry = datetime.fromisoformat(expiry.rstrip("Z")) + cred.storage = storage + return cred + except (AttributeError, KeyError, TypeError, ValueError) as exc: + raise InvalidCredentialsError("Invalid saved credentials") from exc + + +class FileStorage: + def __init__(self, filename): + self.path = Path(filename) + self.lock = threading.RLock() + + def _check_path(self): + if self.path.is_symlink(): + raise InvalidCredentialsError( + "Credentials file cannot be symbolic link" + ) + + def get(self): + with self.lock: + self._check_path() + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + try: + fd = os.open(self.path, flags) + except FileNotFoundError: + return None + with os.fdopen(fd, encoding="utf-8") as stream: + return load_credentials(stream.read(), self) + + def put(self, credentials): + with self.lock: + self._check_path() + self.path.parent.mkdir(parents=True, exist_ok=True) + fd, name = tempfile.mkstemp(dir=self.path.parent) + try: + with os.fdopen(fd, "w", encoding="utf-8") as stream: + stream.write(credentials.to_json()) + os.replace(name, self.path) + finally: + Path(name).unlink(missing_ok=True) + + +class DictionaryStorage: + def __init__(self, dictionary, key): + self.dictionary = dictionary + self.key = key + self.lock = threading.RLock() + + def get(self): + with self.lock: + return load_credentials(self.dictionary.get(self.key), self) + + def put(self, credentials): + with self.lock: + self.dictionary[self.key] = credentials.to_json() diff --git a/pydrive2/auth.py b/pydrive2/auth.py index ce2b185..52e9ae9 100644 --- a/pydrive2/auth.py +++ b/pydrive2/auth.py @@ -1,25 +1,26 @@ +import errno import json import os import threading -import webbrowser from functools import wraps from typing import ClassVar import httplib2 +from google.auth.exceptions import RefreshError as GoogleRefreshError +from google.oauth2 import credentials as user_credentials +from google_auth_httplib2 import AuthorizedHttp, Request +from google_auth_oauthlib.flow import InstalledAppFlow from googleapiclient.discovery import build -from oauth2client import clientsecrets -from oauth2client._helpers import scopes_to_string -from oauth2client.client import ( - OOB_CALLBACK_URN, - AccessTokenRefreshError, - FlowExchangeError, - OAuth2WebServerFlow, +from oauthlib.oauth2 import AccessDeniedError, OAuth2Error + +from ._credentials import ( + DictionaryStorage, + FileStorage, + InvalidCredentialsError, + ServiceCredentials, + UserCredentials, + p12_key_info, ) -from oauth2client.contrib.dictionary_storage import DictionaryStorage -from oauth2client.file import Storage -from oauth2client.service_account import ServiceAccountCredentials -from oauth2client.tools import ClientRedirectHandler, ClientRedirectServer - from .apiattr import ApiAttribute, ApiAttributeMixin from .settings import ( InvalidConfigError, @@ -33,10 +34,6 @@ class AuthError(Exception): """Base error for authentication/authorization errors.""" -class InvalidCredentialsError(IOError): - """Error trying to read credentials file.""" - - class AuthenticationRejected(AuthError): """User rejected authentication.""" @@ -59,7 +56,11 @@ def _decorated(self, *args, **kwargs): self.auth = GoogleAuth() # Re-create access token if it expired. if self.auth.access_token_expired: - if getattr(self.auth, "auth_method", False) == "service": + if self.auth.credentials is not None and not isinstance( + self.auth.credentials, user_credentials.Credentials + ): + self.auth.Refresh() + elif self.auth.auth_method == "service": self.auth.ServiceAuth() else: self.auth.LocalWebserverAuth() @@ -82,7 +83,11 @@ def _decorated(self, *args, **kwargs): else: # If HTTP object not specified, create or resuse an HTTP # object from the thread local storage. - if not getattr(self.auth.thread_local, "http", None): + http = getattr(self.auth.thread_local, "http", None) + if http is None or ( + isinstance(http, AuthorizedHttp) + and http.credentials is not self.auth.credentials + ): self.auth.thread_local.http = self.auth.Get_Http_Object() self.http = self.auth.thread_local.http @@ -91,63 +96,8 @@ def _decorated(self, *args, **kwargs): return _decorated -def CheckServiceAuth(decoratee): - """Decorator to authorize service account.""" - - @wraps(decoratee) - def _decorated(self, *args, **kwargs): - self.auth_method = "service" - dirty = False - save_credentials = self.settings.get("save_credentials") - if self.credentials is None and save_credentials: - self.LoadCredentials() - if self.credentials is None: - decoratee(self, *args, **kwargs) - self.Authorize() - dirty = True - elif self.access_token_expired: - self.Refresh() - dirty = True - self.credentials.set_store(self._default_storage) - if dirty and save_credentials: - self.SaveCredentials() - - return _decorated - - -def CheckAuth(decoratee): - """Decorator to check if it requires OAuth2 flow request.""" - - @wraps(decoratee) - def _decorated(self, *args, **kwargs): - dirty = False - code = None - save_credentials = self.settings.get("save_credentials") - if self.credentials is None and save_credentials: - self.LoadCredentials() - if self.flow is None: - self.GetFlow() - if self.credentials is None: - code = decoratee(self, *args, **kwargs) - dirty = True - else: - if self.access_token_expired: - if self.credentials.refresh_token is not None: - self.Refresh() - else: - code = decoratee(self, *args, **kwargs) - dirty = True - if code is not None: - self.Auth(code) - self.credentials.set_store(self._default_storage) - if dirty and save_credentials: - self.SaveCredentials() - - return _decorated - - class GoogleAuth(ApiAttributeMixin): - """Wrapper class for oauth2client library in google-api-python-client. + """Google Drive authentication using google-auth. Loads all settings and credentials from one 'settings.yaml' file and performs common OAuth2.0 related functionality such as authentication @@ -168,7 +118,6 @@ class GoogleAuth(ApiAttributeMixin): "revoke_uri", "redirect_uri", ] - SERVICE_CONFIGS_LIST: ClassVar[list[str]] = ["client_user_email"] settings = ApiAttribute("settings") client_config = ApiAttribute("client_config") flow = ApiAttribute("flow") @@ -211,11 +160,26 @@ def access_token_expired(self): :returns: bool -- True if access token doesn't exist or is expired. """ - if self.credentials is None: - return True - return self.credentials.access_token_expired + return self.credentials is None or not self.credentials.valid + + def _load_saved_credentials(self): + if self.credentials is None and self.settings.get("save_credentials"): + self.LoadCredentials() + + def _use_flow_credentials(self, credentials): + self.credentials = UserCredentials( + token=credentials.token, + refresh_token=credentials.refresh_token, + id_token=credentials.id_token, + token_uri=credentials.token_uri, + client_id=credentials.client_id, + client_secret=credentials.client_secret, + scopes=credentials.scopes, + expiry=credentials.expiry, + granted_scopes=credentials.granted_scopes, + ) + self.credentials.storage = self._default_storage - @CheckAuth def LocalWebserverAuth( self, host_name="localhost", @@ -242,125 +206,91 @@ def LocalWebserverAuth( If not specified, it will listen on the address specified in the host_name parameter. :type bind_addr: str. - :returns: str -- code returned from local web server :raises: AuthenticationRejected, AuthenticationError """ + self._load_saved_credentials() + if not self.access_token_expired: + return + if self.credentials is not None and ( + not isinstance(self.credentials, user_credentials.Credentials) + or self.credentials.refresh_token + ): + self.Refresh() + return if os.getenv("GDRIVE_NON_INTERACTIVE"): raise AuthenticationError( "Non interactive mode (GDRIVE_NON_INTERACTIVE env) is enabled" ) - + if self.flow is None: + self.GetFlow() if port_numbers is None: - port_numbers = [ - 8080, - 8090, - ] # Mutable objects should not be default - # values, as each call's changes are global. - success = False - port_number = 0 + port_numbers = (8080, 8090) for port in port_numbers: - port_number = port try: - httpd = ClientRedirectServer( - (bind_addr or host_name, port), ClientRedirectHandler + credentials = self.flow.run_local_server( + host=host_name, + bind_addr=bind_addr, + port=port, + open_browser=launch_browser, + **self._authorization_params(), ) - except OSError: - pass - else: - success = True - break - if success: - oauth_callback = f"http://{host_name}:{port_number}/" - else: - print( - "Failed to start a local web server. Please check your firewall" - ) - print( - "settings and locally running programs that may be blocking or" - ) - print("using configured ports. Default ports are 8080 and 8090.") - raise AuthenticationError() - self.flow.redirect_uri = oauth_callback - authorize_url = self.GetAuthUrl() - if launch_browser: - webbrowser.open(authorize_url, new=1, autoraise=True) - print("Your browser has been opened to visit:") - else: - print("Open your browser to visit:") - print() - print(" " + authorize_url) - print() - httpd.handle_request() - if "error" in httpd.query_params: - print("Authentication request was rejected") - raise AuthenticationRejected("User rejected authentication") - if "code" in httpd.query_params: - return httpd.query_params["code"] - else: - print( - 'Failed to find "code" in the query parameters of the redirect.' - ) - print("Try command-line authentication") - raise AuthenticationError("No code found in redirect") + except OSError as exc: + if exc.errno not in (errno.EADDRINUSE, errno.EACCES): + raise + continue + except AccessDeniedError as exc: + raise AuthenticationRejected( + "User rejected authentication" + ) from exc + except OAuth2Error as exc: + raise AuthenticationError( + "OAuth2 authorization failed" + ) from exc + self._use_flow_credentials(credentials) + self.Authorize() + if self.settings.get("save_credentials"): + self.SaveCredentials() + return + raise AuthenticationError( + "Could not bind to any configured OAuth callback port" + ) - @CheckAuth def CommandLineAuth(self): - """Authenticate and authorize from user by printing authentication url - retrieving authentication code from command-line. + """Print a login URL and receive the authorization on a local port.""" + return self.LocalWebserverAuth(launch_browser=False) - :returns: str -- code returned from commandline. - """ - self.flow.redirect_uri = OOB_CALLBACK_URN - authorize_url = self.GetAuthUrl() - print("Go to the following link in your browser:") - print() - print(" " + authorize_url) - print() - return input("Enter verification code: ").strip() - - @CheckServiceAuth def ServiceAuth(self): - """Authenticate and authorize using P12 private key, client id - and client email for a Service account. - :raises: AuthError, InvalidConfigError - """ - if set(self.SERVICE_CONFIGS_LIST) - set(self.client_config): + """Authenticate with a service account, optionally acting for a user.""" + self.auth_method = "service" + self._load_saved_credentials() + if self.credentials is None: self.LoadServiceConfigSettings() - scopes = scopes_to_string(self.settings["oauth_scope"]) - keyfile_name = self.client_config.get("client_json_file_path") - keyfile_dict = self.client_config.get("client_json_dict") - keyfile_json = self.client_config.get("client_json") - - if not keyfile_dict and keyfile_json: - # Compensating for missing ServiceAccountCredentials.from_json_keyfile - keyfile_dict = json.loads(keyfile_json) - - if keyfile_dict: - self.credentials = ( - ServiceAccountCredentials.from_json_keyfile_dict( - keyfile_dict=keyfile_dict, scopes=scopes - ) - ) - elif keyfile_name: - self.credentials = ( - ServiceAccountCredentials.from_json_keyfile_name( - filename=keyfile_name, scopes=scopes - ) - ) - else: - service_email = self.client_config["client_service_email"] - file_path = self.client_config["client_pkcs12_file_path"] - self.credentials = ServiceAccountCredentials.from_p12_keyfile( - service_account_email=service_email, - filename=file_path, - scopes=scopes, - ) - - user_email = self.client_config.get("client_user_email") - if user_email: - self.credentials = self.credentials.create_delegated( - sub=user_email + config = self.client_config + if "client_json_dict" in config: + info = config["client_json_dict"] + elif "client_json" in config: + info = json.loads(config["client_json"]) + elif "client_json_file_path" in config: + with open( + config["client_json_file_path"], encoding="utf-8" + ) as stream: + info = json.load(stream) + else: + with open(config["client_pkcs12_file_path"], "rb") as stream: + info = p12_key_info( + stream.read(), config["client_service_email"] + ) + self.credentials = ServiceCredentials.from_key_info( + info, + self.settings["oauth_scope"], + config.get("client_user_email"), ) + if self.settings.get("save_credentials"): + self.credentials.storage = self._default_storage + if self.access_token_expired: + self.Refresh() + if self.service is None: + self.Authorize() def _InitializeStoragesFromSettings(self): result = {"file": None, "dictionary": None} @@ -372,7 +302,7 @@ def _InitializeStoragesFromSettings(self): raise InvalidConfigError( "Please specify credentials file to read" ) - result[backend] = Storage(credentials_file) + result[backend] = FileStorage(credentials_file) elif backend == "dictionary": creds_dict = self.settings.get("save_credentials_dict") if creds_dict is None: @@ -425,17 +355,9 @@ def LoadCredentialsFile(self, credentials_file=None): "file or pass an explicit value" ) else: - self._default_storage = Storage(credentials_file) - - try: - self.credentials = self._default_storage.get() - except OSError: - raise InvalidCredentialsError( - "Credentials file cannot be symbolic link" - ) + self._default_storage = FileStorage(credentials_file) - if self.credentials: - self.credentials.set_store(self._default_storage) + self.credentials = self._default_storage.get() def _LoadCredentialsDictionary(self): self._default_storage = self._storages["dictionary"] @@ -447,9 +369,6 @@ def _LoadCredentialsDictionary(self): self.credentials = self._default_storage.get() - if self.credentials: - self.credentials.set_store(self._default_storage) - def SaveCredentials(self, backend=None): """Saves credentials according to specified backend. @@ -490,14 +409,9 @@ def SaveCredentialsFile(self, credentials_file=None): "file or pass an explicit value" ) else: - storage = Storage(credentials_file) + storage = FileStorage(credentials_file) - try: - storage.put(self.credentials) - except OSError: - raise InvalidCredentialsError( - "Credentials file cannot be symbolic link" - ) + storage.put(self.credentials) def _SaveCredentialsDictionary(self): if self.credentials is None: @@ -549,44 +463,27 @@ def LoadClientConfigFile(self, client_config_file=None): if client_config_file is None: client_config_file = self.settings["client_config_file"] try: - client_type, client_info = clientsecrets.loadfile( - client_config_file - ) - except clientsecrets.InvalidClientSecretsError as error: - raise InvalidConfigError(f"Invalid client secrets file {error}") - if client_type not in ( - clientsecrets.TYPE_WEB, - clientsecrets.TYPE_INSTALLED, - ): - raise InvalidConfigError( - "Unknown client_type of client config file" + with open(client_config_file, encoding="utf-8") as stream: + config = json.load(stream) + (client_type,) = config + if client_type not in ("web", "installed"): + raise ValueError("Expected web or installed client") + info = config[client_type] + self.client_config = { + name: info[name] + for name in ( + "client_id", + "client_secret", + "auth_uri", + "token_uri", + ) + } + self.client_config.update( + redirect_uri=info["redirect_uris"][0], + revoke_uri=info.get("revoke_uri"), ) - - # General settings. - try: - config_index = [ - "client_id", - "client_secret", - "auth_uri", - "token_uri", - ] - for config in config_index: - self.client_config[config] = client_info[config] - - self.client_config["revoke_uri"] = client_info.get("revoke_uri") - self.client_config["redirect_uri"] = client_info["redirect_uris"][ - 0 - ] - except KeyError: - raise InvalidConfigError("Insufficient client config in file") - - # Service auth related fields. - service_auth_config = ["client_email"] - try: - for config in service_auth_config: - self.client_config[config] = client_info[config] - except KeyError: - pass # The service auth fields are not present, handling code can go here. + except (OSError, KeyError, IndexError, TypeError, ValueError) as exc: + raise InvalidConfigError("Invalid client secrets file") from exc def LoadServiceConfigSettings(self): """Loads client configuration from settings. @@ -609,18 +506,18 @@ def LoadServiceConfigSettings(self): f"One of {configs} is required for service authentication" ) + service_config = self.settings["service_config"] + self.client_config["client_user_email"] = service_config.get( + "client_user_email" + ) if config == "client_pkcs12_file_path": - self.SERVICE_CONFIGS_LIST.append("client_service_email") - - for config in self.SERVICE_CONFIGS_LIST: - try: - self.client_config[config] = self.settings["service_config"][ - config - ] - except KeyError: - err = "Insufficient service config in settings" - err += f"\n\nMissing: {config} key." - raise InvalidConfigError(err) + if not service_config.get("client_service_email"): + raise InvalidConfigError( + "Missing client_service_email for PKCS12 key" + ) + self.client_config["client_service_email"] = service_config[ + "client_service_email" + ] def LoadClientConfigSettings(self): """Loads client configuration from settings file. @@ -646,24 +543,20 @@ def GetFlow(self): config in self.client_config for config in self.CLIENT_CONFIGS_LIST ): self.LoadClientConfig() - constructor_kwargs = { - "redirect_uri": self.client_config["redirect_uri"], - "auth_uri": self.client_config["auth_uri"], - "token_uri": self.client_config["token_uri"], - "access_type": "online", - } - if self.client_config["revoke_uri"] is not None: - constructor_kwargs["revoke_uri"] = self.client_config["revoke_uri"] - self.flow = OAuth2WebServerFlow( - self.client_config["client_id"], - self.client_config["client_secret"], - scopes_to_string(self.settings["oauth_scope"]), - **constructor_kwargs, + redirect_uri = self.client_config["redirect_uri"] + if redirect_uri == "urn:ietf:wg:oauth:2.0:oob": + redirect_uri = "http://localhost" + self.flow = InstalledAppFlow.from_client_config( + {"installed": self.client_config}, + scopes=self.settings["oauth_scope"], + redirect_uri=redirect_uri, + autogenerate_code_verifier=True, ) + + def _authorization_params(self): if self.settings.get("get_refresh_token"): - self.flow.params.update( - {"access_type": "offline", "approval_prompt": "force"} - ) + return {"access_type": "offline", "prompt": "consent"} + return {"access_type": "online"} def Refresh(self): """Refreshes the access_token. @@ -673,8 +566,8 @@ def Refresh(self): if self.credentials is None: raise RefreshError("No credential to refresh.") if ( - self.credentials.refresh_token is None - and self.auth_method != "service" + isinstance(self.credentials, user_credentials.Credentials) + and self.credentials.refresh_token is None ): raise RefreshError( "No refresh_token found." @@ -683,8 +576,13 @@ def Refresh(self): if self.http is None: self.http = self._build_http() try: - self.credentials.refresh(self.http) - except AccessTokenRefreshError as error: + http = ( + self.http.http + if isinstance(self.http, AuthorizedHttp) + else self.http + ) + self.credentials.refresh(Request(http)) + except GoogleRefreshError as error: raise RefreshError(f"Access token refresh failed: {error}") def GetAuthUrl(self): @@ -694,7 +592,8 @@ def GetAuthUrl(self): """ if self.flow is None: self.GetFlow() - return self.flow.step1_get_authorize_url() + url, _ = self.flow.authorization_url(**self._authorization_params()) + return url def Auth(self, code): """Authenticate, authorize, and build service. @@ -716,8 +615,9 @@ def Authenticate(self, code): if self.flow is None: self.GetFlow() try: - self.credentials = self.flow.step2_exchange(code) - except FlowExchangeError as e: + self.flow.fetch_token(code=code) + self._use_flow_credentials(self.flow.credentials) + except OAuth2Error as e: raise AuthenticationError(f"OAuth2 code exchange failed: {e}") print("Authentication successful.") @@ -748,7 +648,9 @@ def Authorize(self): if self.http is None: self.http = self._build_http() - self.http = self.credentials.authorize(self.http) + if isinstance(self.http, AuthorizedHttp): + self.http = self.http.http + self.http = AuthorizedHttp(self.credentials, http=self.http) self.service = build( "drive", "v2", http=self.http, cache_discovery=False ) @@ -759,6 +661,4 @@ def Get_Http_Object(self): :return: The http object to be used in each call. :rtype: httplib2.Http """ - http = self._build_http() - http = self.credentials.authorize(http) - return http + return AuthorizedHttp(self.credentials, http=self._build_http()) diff --git a/pydrive2/settings.py b/pydrive2/settings.py index 1b4dc11..f986807 100644 --- a/pydrive2/settings.py +++ b/pydrive2/settings.py @@ -57,7 +57,7 @@ "redirect_uri": { "type": str, "required": True, - "default": "urn:ietf:wg:oauth:2.0:oob", + "default": "http://localhost", }, "revoke_uri": {"type": str, "required": True, "default": None}, }, diff --git a/pydrive2/test/test_auth_migration.py b/pydrive2/test/test_auth_migration.py new file mode 100644 index 0000000..511b44b --- /dev/null +++ b/pydrive2/test/test_auth_migration.py @@ -0,0 +1,335 @@ +import base64 +import errno +import json +import os +from urllib.parse import parse_qs, urlparse + +import httplib2 +import pytest +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives.serialization import pkcs12 +from google.auth import jwt +from google.oauth2.credentials import Credentials +from google_auth_httplib2 import AuthorizedHttp + +from pydrive2.auth import GoogleAuth, InvalidCredentialsError + +TOKEN_URI = "https://oauth2.googleapis.com/token" +EMAIL = "service@example.test" + + +@pytest.fixture(scope="module") +def key(): + return rsa.generate_private_key(public_exponent=65537, key_size=2048) + + +@pytest.fixture +def service_info(key): + return { + "type": "service_account", + "client_email": EMAIL, + "private_key_id": "test-key", + "token_uri": TOKEN_URI, + "private_key": key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ).decode(), + } + + +@pytest.fixture +def token_endpoint(monkeypatch): + calls = [] + + def request(http, uri, method="GET", body=None, **kwargs): + assert (uri, method) == (TOKEN_URI, "POST") + calls.append( + parse_qs(body.decode() if isinstance(body, bytes) else body) + ) + return ( + httplib2.Response({"status": "200"}), + json.dumps( + { + "access_token": "fresh", + "expires_in": 3600, + "token_type": "Bearer", + } + ).encode(), + ) + + monkeypatch.setattr(httplib2.Http, "request", request) + return calls + + +def config(backend): + return { + "client_config_backend": "settings", + "client_config": {"client_id": "client", "client_secret": "secret"}, + "save_credentials": True, + "save_credentials_backend": backend, + } + + +@pytest.mark.parametrize("backend", ["dictionary", "file"]) +def test_legacy_user_cache_refresh(backend, tmp_path, token_endpoint): + old = json.dumps( + { + "_class": "OAuth2Credentials", + "_module": "oauth2client.client", + "access_token": "stale", + "refresh_token": "refresh", + "token_expiry": "2000-01-01T00:00:00Z", + "token_uri": TOKEN_URI, + "client_id": "client", + "client_secret": "secret", + "scopes": ["drive"], + } + ) + settings = config(backend) + if backend == "dictionary": + settings.update( + save_credentials_dict={"user": old}, save_credentials_key="user" + ) + else: + path = tmp_path / "credentials" + path.write_text(old) + settings["save_credentials_file"] = str(path) + auth = GoogleAuth(settings=settings) + auth.LocalWebserverAuth() + assert auth.credentials.token == "fresh" + assert token_endpoint[0]["refresh_token"] == ["refresh"] + reloaded = GoogleAuth(settings=settings) + reloaded.CommandLineAuth() + assert reloaded.credentials.valid + assert reloaded.credentials.refresh_token == "refresh" + assert len(token_endpoint) == 1 + if backend == "file" and os.name == "posix": + assert path.stat().st_mode & 0o777 == 0o600 + + +def test_automatic_http_refresh_is_saved(monkeypatch): + saved = { + "user": json.dumps( + { + "token": "old", + "refresh_token": "refresh", + "client_id": "id", + "client_secret": "secret", + "token_uri": TOKEN_URI, + } + ) + } + auth = GoogleAuth( + settings=dict( + config("dictionary"), + save_credentials_dict=saved, + save_credentials_key="user", + ) + ) + auth.LoadCredentials() + http = auth.Get_Http_Object() + + def request(uri, method="GET", headers=None, **kwargs): + if uri == TOKEN_URI: + return ( + httplib2.Response({"status": "200"}), + json.dumps( + { + "access_token": "new", + "refresh_token": "rotated", + "expires_in": 3600, + } + ).encode(), + ) + status = "200" if headers["authorization"] == "Bearer new" else "401" + return httplib2.Response({"status": status}), b"{}" + + monkeypatch.setattr(http.http, "request", request) + assert http.request("https://example.test/drive")[0].status == 200 + assert json.loads(saved["user"])["refresh_token"] == "rotated" + + +@pytest.mark.parametrize( + "source", + [ + "client_json", + "client_json_dict", + "client_json_file_path", + "client_pkcs12_file_path", + ], +) +def test_service_account_signing_and_cache( + source, service_info, key, tmp_path, token_endpoint +): + config = {"client_user_email": "delegate@example.test"} + if source == "client_json_dict": + config[source] = service_info + elif source == "client_json": + config[source] = json.dumps(service_info) + else: + keyfile = tmp_path / "key" + if source == "client_pkcs12_file_path": + keyfile.write_bytes( + pkcs12.serialize_key_and_certificates( + b"key", + key, + None, + None, + serialization.BestAvailableEncryption(b"notasecret"), + ) + ) + config["client_service_email"] = EMAIL + else: + keyfile.write_text(json.dumps(service_info)) + config[source] = str(keyfile) + settings = { + "client_config_backend": "service", + "service_config": config, + "oauth_scope": ["scope-a", "scope-b"], + "save_credentials": True, + "save_credentials_backend": "dictionary", + "save_credentials_dict": {}, + "save_credentials_key": "account", + } + auth = GoogleAuth(settings=settings) + auth.ServiceAuth() + assertion = token_endpoint[0]["assertion"][0] + claims = jwt.decode(assertion, verify=False) + assert (claims["iss"], claims["sub"], claims["scope"]) == ( + EMAIL, + "delegate@example.test", + "scope-a scope-b", + ) + assert auth.credentials.valid + assert isinstance(auth.http, AuthorizedHttp) + assert auth.http.http is not auth.Get_Http_Object().http + assert 308 not in auth.http.http.redirect_codes + settings["service_config"] = { + "client_json_file_path": str(tmp_path / "missing") + } + reloaded = GoogleAuth(settings=settings) + reloaded.ServiceAuth() + assert reloaded.credentials.valid + assert len(token_endpoint) == 1 + reloaded.Refresh() + assert len(token_endpoint) == 2 + + +@pytest.mark.parametrize("use_p12", [False, True]) +def test_legacy_service_cache( + use_p12, service_info, key, tmp_path, token_endpoint +): + record = { + "_class": "ServiceAccountCredentials", + "_service_account_email": EMAIL, + "_scopes": "drive", + "_kwargs": {"sub": "delegate@example.test"}, + "token_uri": TOKEN_URI, + "access_token": None, + } + if use_p12: + p12 = pkcs12.serialize_key_and_certificates( + b"key", + key, + None, + None, + serialization.BestAvailableEncryption(b"notasecret"), + ) + record["_private_key_pkcs12"] = base64.b64encode(p12).decode() + else: + record["_private_key_pkcs8_pem"] = service_info["private_key"] + path = tmp_path / "cache" + path.write_text(json.dumps(record)) + auth = GoogleAuth(settings_file=None) + auth.LoadCredentialsFile(path) + auth.ServiceAuth() + assert auth.credentials.valid + assert json.loads(path.read_text())["token"] == "fresh" + assert ( + jwt.decode(token_endpoint[0]["assertion"][0], verify=False)["sub"] + == "delegate@example.test" + ) + + +def test_login_options_and_port_fallback(mocker): + auth = GoogleAuth( + settings={ + "client_config_backend": "settings", + "get_refresh_token": True, + "client_config": {"client_id": "id", "client_secret": "secret"}, + } + ) + query = parse_qs(urlparse(auth.GetAuthUrl()).query) + assert query["access_type"] == ["offline"] + assert query["prompt"] == ["consent"] + assert query["code_challenge_method"] == ["S256"] + assert query["state"] + login = mocker.patch.object( + auth.flow, + "run_local_server", + side_effect=[ + OSError(errno.EADDRINUSE, "busy"), + Credentials("token"), + ], + ) + auth.LocalWebserverAuth( + port_numbers=[8080, 8090], + launch_browser=False, + host_name="localhost", + bind_addr="127.0.0.1", + ) + assert [call.kwargs["port"] for call in login.call_args_list] == [ + 8080, + 8090, + ] + assert login.call_args.kwargs["open_browser"] is False + assert login.call_args.kwargs["bind_addr"] == "127.0.0.1" + assert auth.service is not None + + +def test_custom_auth_and_online_only_cache(tmp_path, mocker): + auth = GoogleAuth( + settings={ + "client_config_backend": "settings", + "client_config": { + "client_id": "id", + "client_secret": "secret", + "redirect_uri": "https://example.test/callback", + }, + } + ) + query = parse_qs(urlparse(auth.GetAuthUrl()).query) + assert query["access_type"] == ["online"] + assert query["redirect_uri"] == ["https://example.test/callback"] + mocker.patch.object(auth.flow, "fetch_token") + auth.flow.oauth2session.token = { + "access_token": "online", + "expires_at": 4102444800, + } + auth.Auth("code") + auth.flow.fetch_token.assert_called_once_with(code="code") + path = tmp_path / "online" + auth.SaveCredentialsFile(path) + other = GoogleAuth(settings_file=None) + other.LoadCredentialsFile(path) + assert other.credentials.valid + assert other.credentials.refresh_token is None + + +def test_credentials_file_rejects_symlinks(tmp_path): + auth = GoogleAuth(settings_file=None) + auth.credentials = Credentials("test") + link = tmp_path / "link" + target = tmp_path / "target" + target.write_text("original") + try: + link.symlink_to(target) + except OSError: + pytest.skip("Symlinks unavailable") + with pytest.raises(InvalidCredentialsError): + auth.SaveCredentialsFile(link) + with pytest.raises(InvalidCredentialsError): + auth.LoadCredentialsFile(link) + assert target.read_text() == "original" diff --git a/pydrive2/test/test_oauth.py b/pydrive2/test/test_oauth.py index 3aa5a12..2768d18 100644 --- a/pydrive2/test/test_oauth.py +++ b/pydrive2/test/test_oauth.py @@ -4,8 +4,8 @@ import time import pytest -from oauth2client.file import Storage +from pydrive2._credentials import FileStorage from pydrive2.auth import AuthenticationError, GoogleAuth from pydrive2.test.test_util import ( GDRIVE_USER_CREDENTIALS_DATA, @@ -126,7 +126,7 @@ def test_09_SaveLoadCredentialsUsesDefaultStorage(mocker): # Delete old credentials file delete_file(credentials_file) assert not os.path.exists(credentials_file) - spy = mocker.spy(Storage, "__init__") + spy = mocker.spy(FileStorage, "__init__") ga.ServiceAuth() ga.LoadCredentials() ga.SaveCredentials() diff --git a/pyproject.toml b/pyproject.toml index 5c53e55..9f5772a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,11 +24,13 @@ classifiers = [ requires-python = ">=3.11" dependencies = [ "google-api-python-client>=1.12.5", - "oauth2client>=4.0.0", + "google-auth>=2.15.0", + "google-auth-httplib2>=0.2.0", + "google-auth-oauthlib>=1.2.0", + "httplib2>=0.18.0", + "oauthlib>=3.0.0", "PyYAML>=3.0", - # https://github.com/iterative/PyDrive2/issues/361 - "cryptography<44", - "pyOpenSSL>=19.1.0,<=24.2.1", + "cryptography>=3.4", ] license = {text = "Apache License 2.0"} dynamic = ["version"] From 47c6eca2dbe8d4718c0ec2dcbac7b53ca3e8d418 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Thu, 19 Nov 2026 16:03:51 -0800 Subject: [PATCH 02/15] Require a fresh OAuth login for PyDrive2 2.x --- docs/fsspec.rst | 6 + docs/oauth.rst | 22 ++-- pydrive2/_credentials.py | 121 +++--------------- pydrive2/auth.py | 90 ++++++++----- pydrive2/fs/spec.py | 5 +- .../test/settings/test_oauth_test_06.yaml | 11 -- .../test/settings/test_oauth_test_07.yaml | 10 -- .../test/settings/test_oauth_test_09.yaml | 10 -- ...t_auth_migration.py => test_auth_flows.py} | 114 ++--------------- pydrive2/test/test_oauth.py | 86 +++---------- 10 files changed, 128 insertions(+), 347 deletions(-) delete mode 100644 pydrive2/test/settings/test_oauth_test_06.yaml delete mode 100644 pydrive2/test/settings/test_oauth_test_07.yaml delete mode 100644 pydrive2/test/settings/test_oauth_test_09.yaml rename pydrive2/test/{test_auth_migration.py => test_auth_flows.py} (69%) diff --git a/docs/fsspec.rst b/docs/fsspec.rst index 4216b96..4714102 100644 --- a/docs/fsspec.rst +++ b/docs/fsspec.rst @@ -39,6 +39,12 @@ someone else's cached credentials: profile="myprofile", ) +PyDrive2 2.x uses a separate default cache. The first interactive connection +after upgrading from 1.x prompts you to sign in. If you supplied a user +credential file explicitly, choose a fresh path. For DVC jobs that use +``GDRIVE_CREDENTIALS_DATA`` for a user account, replace that secret with the +newly saved JSON. Service-account jobs can reuse their configured key. + Writing cached credentials to a file and using it if it already exists (which avoids interactive auth): diff --git a/docs/oauth.rst b/docs/oauth.rst index 091e43a..eba60dc 100644 --- a/docs/oauth.rst +++ b/docs/oauth.rst @@ -97,7 +97,7 @@ Fields explained: :service_config['client_json_file_path'] (str): Path to service account `.json` key file. **Required**: No. :service_config['client_json_dict'] (dict): Service account `.json` key file loaded into a dictionary. **Required**: No. :service_config['client_json'] (str): Service account `.json` key file loaded into a string. **Required**: No. -:save_credentials (bool): True if you want to save credentials. **Default**: False. **Required**: No. +:save_credentials (bool): Save OAuth user credentials after signing in. **Default**: False. **Required**: No. :save_credentials_backend (str): Backend to save credentials to. 'file' and 'dictionary' are the only valid values for now. **Default**: 'file'. **Required**: No. :save_credentials_file (str): Destination of credentials file. **Required**: Yes, only if *save_credentials_backend* is 'file'. :save_credentials_dict (dict): Dict to use for storing credentials. **Required**: Yes, only if *save_credentials_backend* is 'dictionary'. @@ -107,11 +107,18 @@ Fields explained: .. _delegated: https://developers.google.com/admin-sdk/directory/v1/guides/delegation -PyDrive2 reads existing credentials saved by ``oauth2client`` from either -backend. After a refresh or explicit save, it writes the new format. Back up -the cache first if an older PyDrive2 installation also uses it. An explicitly -saved service-account credential includes the signing key so it can refresh -independently of the original key file. +Upgrading from PyDrive2 1.x +___________________________ + +Sign in once with PyDrive2 2.x to create new OAuth user credentials. Keep +your existing OAuth client ID and client secret. If you configured a cache, +choose a new ``save_credentials_file`` or ``save_credentials_key`` before +signing in. For unattended jobs, set ``get_refresh_token: True`` during +sign-in and provision the newly saved credentials before upgrading the job. + +Service accounts authenticate directly with their configured JSON or PKCS12 +key. Keep that key available to the application. If your deployment relied +solely on a saved 1.x service-account cache, provision its signing key. Sample *settings.yaml* ______________________ @@ -193,8 +200,7 @@ If we use OAuth client ID we need to do one manual login into the account with :return: google auth """ # Define the settings dict to use a service account - # We also can use all options available for the settings dict like - # oauth_scope,save_credentials,etc. + # Set oauth_scope here to request specific Drive permissions. settings = { "client_config_backend": "service", "service_config": { diff --git a/pydrive2/_credentials.py b/pydrive2/_credentials.py index 7219b3e..f2d7856 100644 --- a/pydrive2/_credentials.py +++ b/pydrive2/_credentials.py @@ -1,23 +1,19 @@ """Credential persistence for GoogleAuth's file and dictionary backends.""" -import base64 import json import os import tempfile import threading -from datetime import datetime from pathlib import Path -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.serialization import pkcs12 -from google.oauth2 import credentials, service_account +from google.oauth2.credentials import Credentials class InvalidCredentialsError(IOError): """Error trying to read or write credentials.""" -class _PersistentCredentials: +class UserCredentials(Credentials): storage = None def refresh(self, request): @@ -34,108 +30,23 @@ def refresh(self, request): storage.put(self) -class UserCredentials(_PersistentCredentials, credentials.Credentials): - pass - - -class ServiceCredentials(_PersistentCredentials, service_account.Credentials): - @classmethod - def from_key_info(cls, info, scopes, subject): - info = dict(info) - info.setdefault("token_uri", "https://oauth2.googleapis.com/token") - cred = cls.from_service_account_info( - info, scopes=scopes, subject=subject - ) - # An explicitly saved service credential must be able to sign again - # even if the original key file is no longer present. - cred._saved_key_info = info - cred._saved_subject = subject - return cred - - def to_json(self): - return json.dumps( - { - "type": "service_account", - "key": self._saved_key_info, - "scopes": self.scopes, - "subject": self._saved_subject, - "token": self.token, - "expiry": ( - self.expiry.isoformat() + "Z" if self.expiry else None - ), - } - ) - - -def p12_key_info(data, email, password=b"notasecret"): - key, _, _ = pkcs12.load_key_and_certificates(data, password) - if key is None: - raise InvalidCredentialsError("No private key in PKCS12 file") - pem = key.private_bytes( - serialization.Encoding.PEM, - serialization.PrivateFormat.PKCS8, - serialization.NoEncryption(), - ) - return { - "client_email": email, - "private_key": pem.decode("ascii"), - "token_uri": "https://oauth2.googleapis.com/token", - } - - -def load_credentials(serialized, storage=None): +def load_credentials(serialized, storage): if not serialized: return None try: - data = json.loads(serialized) - scopes = data.get("scopes") - if data.get("_class") == "ServiceAccountCredentials": - # Migrate credentials previously written by oauth2client. - email = data["_service_account_email"] - if data.get("_private_key_pkcs12"): - info = p12_key_info( - base64.b64decode(data["_private_key_pkcs12"]), - email, - ( - data.get("_private_key_password") or "notasecret" - ).encode(), - ) - else: - info = { - "client_email": email, - "private_key": data["_private_key_pkcs8_pem"], - "private_key_id": data.get("_private_key_id"), - } - info["token_uri"] = data["token_uri"] - cred = ServiceCredentials.from_key_info( - info, data["_scopes"].split(), data["_kwargs"].get("sub") - ) - elif data.get("type") == "service_account": - cred = ServiceCredentials.from_key_info( - data["key"], scopes, data.get("subject") - ) - else: - cred = UserCredentials( - token=None, - refresh_token=data.get("refresh_token"), - token_uri=data.get( - "token_uri", "https://oauth2.googleapis.com/token" - ), - client_id=data["client_id"], - client_secret=data["client_secret"], - scopes=scopes, - rapt_token=data.get("rapt_token"), - quota_project_id=data.get("quota_project_id"), - ) - if not data.get("invalid"): - cred.token = data.get("token", data.get("access_token")) - expiry = data.get("expiry", data.get("token_expiry")) - if expiry: - cred.expiry = datetime.fromisoformat(expiry.rstrip("Z")) - cred.storage = storage - return cred - except (AttributeError, KeyError, TypeError, ValueError) as exc: - raise InvalidCredentialsError("Invalid saved credentials") from exc + info = json.loads(serialized) + if "token" not in info and info.get("type") != "authorized_user": + raise ValueError("Expected google-auth user credentials") + # Online-only logins have an access token but no refresh token. + info.setdefault("refresh_token", None) + cred = UserCredentials.from_authorized_user_info(info) + except (AttributeError, TypeError, ValueError) as exc: + raise InvalidCredentialsError( + "Select a fresh user credential file or dictionary key and sign " + "in again with PyDrive2 2.x." + ) from exc + cred.storage = storage + return cred class FileStorage: diff --git a/pydrive2/auth.py b/pydrive2/auth.py index 52e9ae9..0b2c83d 100644 --- a/pydrive2/auth.py +++ b/pydrive2/auth.py @@ -2,12 +2,15 @@ import json import os import threading +import webbrowser from functools import wraps from typing import ClassVar import httplib2 +from cryptography.hazmat.primitives.serialization import pkcs12 +from google.auth.crypt import RSASigner from google.auth.exceptions import RefreshError as GoogleRefreshError -from google.oauth2 import credentials as user_credentials +from google.oauth2 import credentials as user_credentials, service_account from google_auth_httplib2 import AuthorizedHttp, Request from google_auth_oauthlib.flow import InstalledAppFlow from googleapiclient.discovery import build @@ -17,9 +20,7 @@ DictionaryStorage, FileStorage, InvalidCredentialsError, - ServiceCredentials, UserCredentials, - p12_key_info, ) from .apiattr import ApiAttribute, ApiAttributeMixin from .settings import ( @@ -162,10 +163,6 @@ def access_token_expired(self): """ return self.credentials is None or not self.credentials.valid - def _load_saved_credentials(self): - if self.credentials is None and self.settings.get("save_credentials"): - self.LoadCredentials() - def _use_flow_credentials(self, credentials): self.credentials = UserCredentials( token=credentials.token, @@ -208,7 +205,8 @@ def LocalWebserverAuth( :type bind_addr: str. :raises: AuthenticationRejected, AuthenticationError """ - self._load_saved_credentials() + if self.credentials is None and self.settings.get("save_credentials"): + self.LoadCredentials() if not self.access_token_expired: return if self.credentials is not None and ( @@ -225,6 +223,11 @@ def LocalWebserverAuth( self.GetFlow() if port_numbers is None: port_numbers = (8080, 8090) + if launch_browser: + try: + webbrowser.get() + except webbrowser.Error: + launch_browser = False for port in port_numbers: try: credentials = self.flow.run_local_server( @@ -262,31 +265,48 @@ def CommandLineAuth(self): def ServiceAuth(self): """Authenticate with a service account, optionally acting for a user.""" self.auth_method = "service" - self._load_saved_credentials() - if self.credentials is None: + if not isinstance(self.credentials, service_account.Credentials): self.LoadServiceConfigSettings() config = self.client_config - if "client_json_dict" in config: - info = config["client_json_dict"] - elif "client_json" in config: - info = json.loads(config["client_json"]) - elif "client_json_file_path" in config: - with open( - config["client_json_file_path"], encoding="utf-8" - ) as stream: - info = json.load(stream) - else: + options = { + "scopes": self.settings["oauth_scope"], + "subject": config.get("client_user_email"), + } + if "client_pkcs12_file_path" in config: with open(config["client_pkcs12_file_path"], "rb") as stream: - info = p12_key_info( - stream.read(), config["client_service_email"] + key, _, _ = pkcs12.load_key_and_certificates( + stream.read(), b"notasecret" ) - self.credentials = ServiceCredentials.from_key_info( - info, - self.settings["oauth_scope"], - config.get("client_user_email"), - ) - if self.settings.get("save_credentials"): - self.credentials.storage = self._default_storage + if key is None: + raise InvalidCredentialsError( + "No private key in PKCS12 file" + ) + self.credentials = service_account.Credentials( + RSASigner(key), + config["client_service_email"], + "https://oauth2.googleapis.com/token", + **options, + ) + else: + if "client_json_dict" in config: + info = config["client_json_dict"] + elif "client_json" in config: + info = json.loads(config["client_json"]) + else: + with open( + config["client_json_file_path"], encoding="utf-8" + ) as stream: + info = json.load(stream) + info = { + "token_uri": "https://oauth2.googleapis.com/token", + **info, + } + self.credentials = ( + service_account.Credentials.from_service_account_info( + info, **options + ) + ) + self.service = None if self.access_token_expired: self.Refresh() if self.service is None: @@ -397,8 +417,10 @@ def SaveCredentialsFile(self, credentials_file=None): :type credentials_file: str. :raises: InvalidConfigError, InvalidCredentialsError """ - if self.credentials is None: - raise InvalidCredentialsError("No credentials to save") + if not isinstance(self.credentials, user_credentials.Credentials): + raise InvalidCredentialsError( + "Only signed-in user credentials can be saved" + ) if credentials_file is None: storage = self._storages["file"] @@ -414,8 +436,10 @@ def SaveCredentialsFile(self, credentials_file=None): storage.put(self.credentials) def _SaveCredentialsDictionary(self): - if self.credentials is None: - raise InvalidCredentialsError("No credentials to save") + if not isinstance(self.credentials, user_credentials.Credentials): + raise InvalidCredentialsError( + "Only signed-in user credentials can be saved" + ) storage = self._storages["dictionary"] if storage is None: diff --git a/pydrive2/fs/spec.py b/pydrive2/fs/spec.py index 6a2b062..2cee435 100644 --- a/pydrive2/fs/spec.py +++ b/pydrive2/fs/spec.py @@ -14,7 +14,7 @@ from funcy import cached_property, retry, wrap_prop, wrap_with from tqdm.utils import CallbackIOWrapper -from pydrive2.auth import GoogleAuth +from pydrive2.auth import GoogleAuth, InvalidCredentialsError from pydrive2.drive import GoogleDrive from pydrive2.fs.utils import IterStream @@ -74,6 +74,8 @@ def should_retry(exc): def _wrap_errors(): try: yield + except InvalidCredentialsError as exc: + raise GDriveAuthError(str(exc)) from exc except Exception as exc: # Handle AuthenticationError, RefreshError and other auth failures # It's hard to come up with a narrow exception, since PyDrive throws @@ -100,6 +102,7 @@ def _client_auth( if not creds_file: cache_dir = os.path.join( appdirs.user_cache_dir("pydrive2fs", appauthor=False), + "v2", client_id, ) os.makedirs(cache_dir, exist_ok=True) diff --git a/pydrive2/test/settings/test_oauth_test_06.yaml b/pydrive2/test/settings/test_oauth_test_06.yaml deleted file mode 100644 index 8ecb81b..0000000 --- a/pydrive2/test/settings/test_oauth_test_06.yaml +++ /dev/null @@ -1,11 +0,0 @@ -client_config_backend: service -service_config: - client_service_email: your-service-account-email - client_pkcs12_file_path: your-file-path.p12 - -save_credentials: True -save_credentials_backend: file -save_credentials_file: credentials/6.dat - -oauth_scope: - - https://www.googleapis.com/auth/drive diff --git a/pydrive2/test/settings/test_oauth_test_07.yaml b/pydrive2/test/settings/test_oauth_test_07.yaml deleted file mode 100644 index a806515..0000000 --- a/pydrive2/test/settings/test_oauth_test_07.yaml +++ /dev/null @@ -1,10 +0,0 @@ -client_config_backend: service -service_config: - client_json_file_path: /tmp/pydrive2/credentials.json - -save_credentials: True -save_credentials_backend: file -save_credentials_file: credentials/7.dat - -oauth_scope: - - https://www.googleapis.com/auth/drive diff --git a/pydrive2/test/settings/test_oauth_test_09.yaml b/pydrive2/test/settings/test_oauth_test_09.yaml deleted file mode 100644 index 09eca82..0000000 --- a/pydrive2/test/settings/test_oauth_test_09.yaml +++ /dev/null @@ -1,10 +0,0 @@ -client_config_backend: service -service_config: - client_json_file_path: /tmp/pydrive2/credentials.json - -save_credentials: True -save_credentials_backend: file -save_credentials_file: credentials/9.dat - -oauth_scope: - - https://www.googleapis.com/auth/drive diff --git a/pydrive2/test/test_auth_migration.py b/pydrive2/test/test_auth_flows.py similarity index 69% rename from pydrive2/test/test_auth_migration.py rename to pydrive2/test/test_auth_flows.py index 511b44b..7f80a1e 100644 --- a/pydrive2/test/test_auth_migration.py +++ b/pydrive2/test/test_auth_flows.py @@ -1,7 +1,6 @@ -import base64 import errno import json -import os +import webbrowser from urllib.parse import parse_qs, urlparse import httplib2 @@ -63,57 +62,12 @@ def request(http, uri, method="GET", body=None, **kwargs): return calls -def config(backend): - return { - "client_config_backend": "settings", - "client_config": {"client_id": "client", "client_secret": "secret"}, - "save_credentials": True, - "save_credentials_backend": backend, - } - - -@pytest.mark.parametrize("backend", ["dictionary", "file"]) -def test_legacy_user_cache_refresh(backend, tmp_path, token_endpoint): - old = json.dumps( - { - "_class": "OAuth2Credentials", - "_module": "oauth2client.client", - "access_token": "stale", - "refresh_token": "refresh", - "token_expiry": "2000-01-01T00:00:00Z", - "token_uri": TOKEN_URI, - "client_id": "client", - "client_secret": "secret", - "scopes": ["drive"], - } - ) - settings = config(backend) - if backend == "dictionary": - settings.update( - save_credentials_dict={"user": old}, save_credentials_key="user" - ) - else: - path = tmp_path / "credentials" - path.write_text(old) - settings["save_credentials_file"] = str(path) - auth = GoogleAuth(settings=settings) - auth.LocalWebserverAuth() - assert auth.credentials.token == "fresh" - assert token_endpoint[0]["refresh_token"] == ["refresh"] - reloaded = GoogleAuth(settings=settings) - reloaded.CommandLineAuth() - assert reloaded.credentials.valid - assert reloaded.credentials.refresh_token == "refresh" - assert len(token_endpoint) == 1 - if backend == "file" and os.name == "posix": - assert path.stat().st_mode & 0o777 == 0o600 - - def test_automatic_http_refresh_is_saved(monkeypatch): saved = { "user": json.dumps( { "token": "old", + "expiry": "2099-01-01T00:00:00Z", "refresh_token": "refresh", "client_id": "id", "client_secret": "secret", @@ -122,11 +76,13 @@ def test_automatic_http_refresh_is_saved(monkeypatch): ) } auth = GoogleAuth( - settings=dict( - config("dictionary"), - save_credentials_dict=saved, - save_credentials_key="user", - ) + settings={ + "client_config_backend": "file", + "save_credentials": True, + "save_credentials_backend": "dictionary", + "save_credentials_dict": saved, + "save_credentials_key": "user", + } ) auth.LoadCredentials() http = auth.Get_Http_Object() @@ -160,7 +116,7 @@ def request(uri, method="GET", headers=None, **kwargs): "client_pkcs12_file_path", ], ) -def test_service_account_signing_and_cache( +def test_service_account_signing( source, service_info, key, tmp_path, token_endpoint ): config = {"client_user_email": "delegate@example.test"} @@ -206,51 +162,9 @@ def test_service_account_signing_and_cache( assert isinstance(auth.http, AuthorizedHttp) assert auth.http.http is not auth.Get_Http_Object().http assert 308 not in auth.http.http.redirect_codes - settings["service_config"] = { - "client_json_file_path": str(tmp_path / "missing") - } - reloaded = GoogleAuth(settings=settings) - reloaded.ServiceAuth() - assert reloaded.credentials.valid - assert len(token_endpoint) == 1 - reloaded.Refresh() - assert len(token_endpoint) == 2 - - -@pytest.mark.parametrize("use_p12", [False, True]) -def test_legacy_service_cache( - use_p12, service_info, key, tmp_path, token_endpoint -): - record = { - "_class": "ServiceAccountCredentials", - "_service_account_email": EMAIL, - "_scopes": "drive", - "_kwargs": {"sub": "delegate@example.test"}, - "token_uri": TOKEN_URI, - "access_token": None, - } - if use_p12: - p12 = pkcs12.serialize_key_and_certificates( - b"key", - key, - None, - None, - serialization.BestAvailableEncryption(b"notasecret"), - ) - record["_private_key_pkcs12"] = base64.b64encode(p12).decode() - else: - record["_private_key_pkcs8_pem"] = service_info["private_key"] - path = tmp_path / "cache" - path.write_text(json.dumps(record)) - auth = GoogleAuth(settings_file=None) - auth.LoadCredentialsFile(path) - auth.ServiceAuth() - assert auth.credentials.valid - assert json.loads(path.read_text())["token"] == "fresh" - assert ( - jwt.decode(token_endpoint[0]["assertion"][0], verify=False)["sub"] - == "delegate@example.test" - ) + assert not settings["save_credentials_dict"] + with pytest.raises(InvalidCredentialsError): + auth.SaveCredentials() def test_login_options_and_port_fallback(mocker): @@ -266,6 +180,7 @@ def test_login_options_and_port_fallback(mocker): assert query["prompt"] == ["consent"] assert query["code_challenge_method"] == ["S256"] assert query["state"] + mocker.patch("webbrowser.get", side_effect=webbrowser.Error("No browser")) login = mocker.patch.object( auth.flow, "run_local_server", @@ -276,7 +191,6 @@ def test_login_options_and_port_fallback(mocker): ) auth.LocalWebserverAuth( port_numbers=[8080, 8090], - launch_browser=False, host_name="localhost", bind_addr="127.0.0.1", ) diff --git a/pydrive2/test/test_oauth.py b/pydrive2/test/test_oauth.py index 2768d18..125de27 100644 --- a/pydrive2/test/test_oauth.py +++ b/pydrive2/test/test_oauth.py @@ -4,6 +4,7 @@ import time import pytest +from google.oauth2.credentials import Credentials from pydrive2._credentials import FileStorage from pydrive2.auth import AuthenticationError, GoogleAuth @@ -78,34 +79,6 @@ def test_05_ConfigFromSettingsWithoutOauthScope(): time.sleep(1) -@pytest.mark.skip(reason="P12 authentication is deprecated") -@pytest.mark.drive -def test_06_ServiceAuthFromSavedCredentialsP12File(): - setup_credentials("credentials/6.dat") - ga = GoogleAuth(settings_file_path("test_oauth_test_06.yaml")) - ga.ServiceAuth() - assert not ga.access_token_expired - time.sleep(1) - - -@pytest.mark.drive -def test_07_ServiceAuthFromSavedCredentialsJsonFile(): - # Have an initial auth so that credentials/7.dat gets saved - ga = GoogleAuth(settings_file_path("test_oauth_test_07.yaml")) - credentials_file = ga.settings["save_credentials_file"] - # Delete old credentials file - delete_file(credentials_file) - assert not os.path.exists(credentials_file) - ga.ServiceAuth() - assert os.path.exists(credentials_file) - # Secondary auth should be made only using the previously saved - # login info - ga = GoogleAuth(settings_file_path("test_oauth_test_07.yaml")) - ga.ServiceAuth() - assert not ga.access_token_expired - time.sleep(1) - - @pytest.mark.drive def test_08_ServiceAuthFromJsonFileNoCredentialsSaving(): # Test that no credentials are saved and API is still functional @@ -116,51 +89,26 @@ def test_08_ServiceAuthFromJsonFileNoCredentialsSaving(): time.sleep(1) -@pytest.mark.drive -def test_09_SaveLoadCredentialsUsesDefaultStorage(mocker): - # Test fix for https://github.com/iterative/PyDrive2/issues/163 - # Make sure that Load and Save credentials by default reuse the - # same Storage (since it defined lock which make it TS) - ga = GoogleAuth(settings_file_path("test_oauth_test_09.yaml")) - credentials_file = ga.settings["save_credentials_file"] - # Delete old credentials file - delete_file(credentials_file) - assert not os.path.exists(credentials_file) +def test_09_SaveLoadCredentialsUsesDefaultStorage(mocker, tmp_path): + # Regression for #163: use the same storage lock for loads and saves. + auth = GoogleAuth( + settings={ + "client_config_backend": "file", + "save_credentials": True, + "save_credentials_backend": "file", + "save_credentials_file": str(tmp_path / "user.json"), + } + ) + auth.credentials = Credentials( + "token", client_id="id", client_secret="secret" + ) spy = mocker.spy(FileStorage, "__init__") - ga.ServiceAuth() - ga.LoadCredentials() - ga.SaveCredentials() + auth.SaveCredentials() + auth.LoadCredentials() + assert auth.credentials.token == "token" assert spy.call_count == 0 -@pytest.mark.drive -def test_10_ServiceAuthFromSavedCredentialsDictionary(): - creds_dict = {} - settings = { - "client_config_backend": "service", - "service_config": { - "client_json_file_path": "/tmp/pydrive2/credentials.json", - }, - "oauth_scope": ["https://www.googleapis.com/auth/drive"], - "save_credentials": True, - "save_credentials_backend": "dictionary", - "save_credentials_dict": creds_dict, - "save_credentials_key": "creds", - } - ga = GoogleAuth(settings=settings) - ga.ServiceAuth() - assert not ga.access_token_expired - assert creds_dict - first_creds_dict = creds_dict.copy() - # Secondary auth should be made only using the previously saved - # login info - ga = GoogleAuth(settings=settings) - ga.ServiceAuth() - assert not ga.access_token_expired - assert creds_dict == first_creds_dict - time.sleep(1) - - @pytest.mark.drive def test_11_ServiceAuthFromJsonNoCredentialsSaving(): client_json = os.environ[GDRIVE_USER_CREDENTIALS_DATA] From 3a59fe74851c2ab16987a63a7fe712777ca5449c Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Thu, 19 Nov 2026 19:22:00 -0800 Subject: [PATCH 03/15] Handle concurrent refresh, refresh-only credentials and OAuth scope changes --- pydrive2/_credentials.py | 8 +++- pydrive2/auth.py | 63 ++++++++++++++---------- pydrive2/test/test_auth_flows.py | 82 +++++++++++++++++++++++++++++++- pyproject.toml | 2 +- 4 files changed, 128 insertions(+), 27 deletions(-) diff --git a/pydrive2/_credentials.py b/pydrive2/_credentials.py index f2d7856..2079367 100644 --- a/pydrive2/_credentials.py +++ b/pydrive2/_credentials.py @@ -35,7 +35,13 @@ def load_credentials(serialized, storage): return None try: info = json.loads(serialized) - if "token" not in info and info.get("type") != "authorized_user": + if ( + not isinstance(info, dict) + or info.get("type", "authorized_user") != "authorized_user" + or "_class" in info + or "access_token" in info + or not (info.get("token") or info.get("refresh_token")) + ): raise ValueError("Expected google-auth user credentials") # Online-only logins have an access token but no refresh token. info.setdefault("refresh_token", None) diff --git a/pydrive2/auth.py b/pydrive2/auth.py index 0b2c83d..b64c286 100644 --- a/pydrive2/auth.py +++ b/pydrive2/auth.py @@ -10,7 +10,8 @@ from cryptography.hazmat.primitives.serialization import pkcs12 from google.auth.crypt import RSASigner from google.auth.exceptions import RefreshError as GoogleRefreshError -from google.oauth2 import credentials as user_credentials, service_account +from google.oauth2 import credentials as user_credentials +from google.oauth2 import service_account from google_auth_httplib2 import AuthorizedHttp, Request from google_auth_oauthlib.flow import InstalledAppFlow from googleapiclient.discovery import build @@ -60,7 +61,7 @@ def _decorated(self, *args, **kwargs): if self.auth.credentials is not None and not isinstance( self.auth.credentials, user_credentials.Credentials ): - self.auth.Refresh() + self.auth._refresh_if_expired() elif self.auth.auth_method == "service": self.auth.ServiceAuth() else: @@ -140,6 +141,7 @@ def __init__( self.http_timeout = http_timeout ApiAttributeMixin.__init__(self) self.thread_local = threading.local() + self._refresh_lock = threading.RLock() self.client_config = {} if settings is None and settings_file: @@ -213,7 +215,7 @@ def LocalWebserverAuth( not isinstance(self.credentials, user_credentials.Credentials) or self.credentials.refresh_token ): - self.Refresh() + self._refresh_if_expired() return if os.getenv("GDRIVE_NON_INTERACTIVE"): raise AuthenticationError( @@ -245,7 +247,7 @@ def LocalWebserverAuth( raise AuthenticationRejected( "User rejected authentication" ) from exc - except OAuth2Error as exc: + except (OAuth2Error, Warning) as exc: raise AuthenticationError( "OAuth2 authorization failed" ) from exc @@ -307,8 +309,7 @@ def ServiceAuth(self): ) ) self.service = None - if self.access_token_expired: - self.Refresh() + self._refresh_if_expired() if self.service is None: self.Authorize() @@ -582,32 +583,44 @@ def _authorization_params(self): return {"access_type": "offline", "prompt": "consent"} return {"access_type": "online"} + def _refresh_if_expired(self): + with self._refresh_lock: + if self.access_token_expired: + self.Refresh() + def Refresh(self): """Refreshes the access_token. :raises: RefreshError """ - if self.credentials is None: - raise RefreshError("No credential to refresh.") - if ( - isinstance(self.credentials, user_credentials.Credentials) - and self.credentials.refresh_token is None - ): - raise RefreshError( - "No refresh_token found." - "Please set access_type of OAuth to offline." - ) - if self.http is None: - self.http = self._build_http() - try: + with self._refresh_lock: + if self.credentials is None: + raise RefreshError("No credential to refresh.") + user = isinstance(self.credentials, user_credentials.Credentials) + if user and self.credentials.refresh_token is None: + raise RefreshError( + "No refresh_token found. " + "Please set access_type of OAuth to offline." + ) + if self.http is None: + self.http = self._build_http() http = ( self.http.http if isinstance(self.http, AuthorizedHttp) else self.http ) - self.credentials.refresh(Request(http)) - except GoogleRefreshError as error: - raise RefreshError(f"Access token refresh failed: {error}") + try: + self.credentials.refresh(Request(http)) + except GoogleRefreshError as exc: + raise RefreshError( + f"Access token refresh failed: {exc}" + ) from exc + if ( + user + and self.settings.get("save_credentials") + and getattr(self.credentials, "storage", None) is None + ): + self.SaveCredentials() def GetAuthUrl(self): """Creates authentication url where user visits to grant access. @@ -641,8 +654,10 @@ def Authenticate(self, code): try: self.flow.fetch_token(code=code) self._use_flow_credentials(self.flow.credentials) - except OAuth2Error as e: - raise AuthenticationError(f"OAuth2 code exchange failed: {e}") + except (OAuth2Error, Warning) as exc: + raise AuthenticationError( + f"OAuth2 code exchange failed: {exc}" + ) from exc print("Authentication successful.") def _build_http(self): diff --git a/pydrive2/test/test_auth_flows.py b/pydrive2/test/test_auth_flows.py index 7f80a1e..94fa1f4 100644 --- a/pydrive2/test/test_auth_flows.py +++ b/pydrive2/test/test_auth_flows.py @@ -1,6 +1,8 @@ import errno import json import webbrowser +from concurrent.futures import ThreadPoolExecutor +from threading import Barrier from urllib.parse import parse_qs, urlparse import httplib2 @@ -12,7 +14,11 @@ from google.oauth2.credentials import Credentials from google_auth_httplib2 import AuthorizedHttp -from pydrive2.auth import GoogleAuth, InvalidCredentialsError +from pydrive2.auth import ( + AuthenticationError, + GoogleAuth, + InvalidCredentialsError, +) TOKEN_URI = "https://oauth2.googleapis.com/token" EMAIL = "service@example.test" @@ -107,6 +113,80 @@ def request(uri, method="GET", headers=None, **kwargs): assert json.loads(saved["user"])["refresh_token"] == "rotated" +def test_refresh_only_credentials_round_trip(tmp_path, token_endpoint): + auth = GoogleAuth(settings_file=None) + auth.credentials = Credentials( + None, + refresh_token="refresh", + client_id="id", + client_secret="secret", + token_uri=TOKEN_URI, + ) + path = tmp_path / "user.json" + auth.SaveCredentialsFile(path) + + other = GoogleAuth(settings_file=None) + other.LoadCredentialsFile(path) + other.LocalWebserverAuth() + assert other.credentials.valid + assert json.loads(path.read_text())["token"] == "fresh" + assert len(token_endpoint) == 1 + + +def test_concurrent_login_refreshes_and_saves_once(mocker, token_endpoint): + saved = {} + auth = GoogleAuth( + settings={ + "client_config_backend": "file", + "save_credentials": True, + "save_credentials_backend": "dictionary", + "save_credentials_dict": saved, + "save_credentials_key": "user", + } + ) + auth.credentials = Credentials( + None, + refresh_token="refresh", + client_id="id", + client_secret="secret", + token_uri=TOKEN_URI, + ) + refresh_if_expired = auth._refresh_if_expired + workers = Barrier(2) + + def refresh_together(): + workers.wait(timeout=10) + refresh_if_expired() + + mocker.patch.object( + auth, "_refresh_if_expired", side_effect=refresh_together + ) + with ThreadPoolExecutor(max_workers=2) as pool: + calls = [pool.submit(auth.LocalWebserverAuth) for _ in range(2)] + for call in calls: + call.result() + + assert len(token_endpoint) == 1 + assert json.loads(saved["user"])["token"] == "fresh" + + +@pytest.mark.parametrize("local_server", [False, True]) +def test_scope_change_raises_authentication_error(mocker, local_server): + auth = GoogleAuth(settings_file=None) + auth.flow = mocker.Mock() + error = Warning("Scope has changed") + if local_server: + auth.flow.run_local_server.side_effect = error + else: + auth.flow.fetch_token.side_effect = error + with pytest.raises(AuthenticationError) as raised: + if local_server: + auth.LocalWebserverAuth(launch_browser=False) + else: + auth.Authenticate("code") + assert raised.value.__cause__ is error + + @pytest.mark.parametrize( "source", [ diff --git a/pyproject.toml b/pyproject.toml index 9f5772a..aa4a991 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ dependencies = [ "google-auth>=2.15.0", "google-auth-httplib2>=0.2.0", "google-auth-oauthlib>=1.2.0", - "httplib2>=0.18.0", + "httplib2>=0.19.0", "oauthlib>=3.0.0", "PyYAML>=3.0", "cryptography>=3.4", From 1608965639c820ea31b00bcbe6037f43fdd1a5c9 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Thu, 19 Nov 2026 19:23:41 -0800 Subject: [PATCH 04/15] Align authentication imports and storage types with the new CI checks --- pydrive2/auth.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pydrive2/auth.py b/pydrive2/auth.py index b64c286..fc02402 100644 --- a/pydrive2/auth.py +++ b/pydrive2/auth.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import errno import json import os @@ -314,7 +316,10 @@ def ServiceAuth(self): self.Authorize() def _InitializeStoragesFromSettings(self): - result = {"file": None, "dictionary": None} + result: dict[str, FileStorage | DictionaryStorage | None] = { + "file": None, + "dictionary": None, + } backend = self.settings.get("save_credentials_backend") save_credentials = self.settings.get("save_credentials") if backend == "file": From 374d1beb1fda5c27e78fc86221c424f467a184af Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Thu, 19 Nov 2026 22:24:37 -0800 Subject: [PATCH 05/15] Wrap reStructuredText consistently and use native annotations --- CONTRIBUTING.rst | 11 ++- README.rst | 45 ++++++----- docs/filelist.rst | 40 +++++---- docs/filemanagement.rst | 79 +++++++++++------- docs/fsspec.rst | 24 ++++-- docs/index.rst | 14 ++-- docs/oauth.rst | 144 ++++++++++++++++++++++++--------- docs/pydrive2.rst | 2 +- docs/quickstart.rst | 169 ++++++++++++++++++++++++--------------- pydrive2/auth.py | 2 - pydrive2/test/README.rst | 97 +++++++++++----------- 11 files changed, 393 insertions(+), 234 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index dc075f6..5a39c51 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -10,11 +10,14 @@ Contributing code ~~~~~~~~~~~~~~~~~ If you have improvements to PyDrive2, send us your pull requests! For those -just getting started, Github has a `howto `_. +just getting started, Github has a `howto`_. -Install the development tools with ``python -m pip install -e '.[tests,fsspec,typing]'``. -Run ``ruff check .`` to lint Python files and ``ruff format .`` to format them. -Run ``python -m mypy`` to type-check the package for Python 3.11. +Install the development tools with +``python -m pip install -e '.[tests,fsspec,typing]'``. Run ``ruff check .`` to +lint Python files and ``ruff format .`` to format them. Run ``python -m mypy`` +to type-check the package for Python 3.11. + +.. _howto: https://help.github.com/articles/using-pull-requests/ Live Drive tests for external pull requests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/README.rst b/README.rst index 8d06442..e3b17fc 100644 --- a/README.rst +++ b/README.rst @@ -1,42 +1,43 @@ |CI| |Conda| |PyPI| -.. |CI| image:: https://github.com/iterative/PyDrive2/workflows/Tests/badge.svg?branch=main +.. |CI| image:: + https://github.com/iterative/PyDrive2/workflows/Tests/badge.svg?branch=main :target: https://github.com/iterative/PyDrive2/actions :alt: GHA Tests -.. |Conda| image:: https://img.shields.io/conda/v/conda-forge/PyDrive2.svg?label=conda&logo=conda-forge +.. |Conda| image:: + https://img.shields.io/conda/v/conda-forge/PyDrive2.svg?label=conda&logo=conda-forge :target: https://anaconda.org/conda-forge/PyDrive2 :alt: Conda-forge -.. |PyPI| image:: https://img.shields.io/pypi/v/PyDrive2.svg?label=pip&logo=PyPI&logoColor=white +.. |PyPI| image:: + https://img.shields.io/pypi/v/PyDrive2.svg?label=pip&logo=PyPI&logoColor=white :target: https://pypi.org/project/PyDrive2 :alt: PyPI PyDrive2 -------- -*PyDrive2* is a wrapper library of -`google-api-python-client `_ -that simplifies many common Google Drive API V2 tasks. It is an actively -maintained fork of `https://pypi.python.org/pypi/PyDrive `_. -By the authors and maintainers of the `Git for Data `_ - DVC -project. +*PyDrive2* is a wrapper library of `google-api-python-client`_ that simplifies +many common Google Drive API V2 tasks. It is an actively maintained fork of +https://pypi.python.org/pypi/PyDrive. By the authors and maintainers of the +`Git for Data `_ - DVC project. Project Info ------------ -- Package: `https://pypi.python.org/pypi/PyDrive2 `_ -- Documentation: `https://docs.iterative.ai/PyDrive2 `_ -- Source: `https://github.com/iterative/PyDrive2 `_ -- Changelog: `https://github.com/iterative/PyDrive2/releases `_ +- Package: https://pypi.python.org/pypi/PyDrive2 +- Documentation: https://docs.iterative.ai/PyDrive2 +- Source: https://github.com/iterative/PyDrive2 +- Changelog: https://github.com/iterative/PyDrive2/releases - `Running tests `_ Features of PyDrive2 -------------------- - Simplifies OAuth2.0 into just few lines with flexible settings. -- Wraps `Google Drive API V2 `_ into - classes of each resource to make your program more object-oriented. +- Wraps `Google Drive API V2`_ into classes of each resource to make your + program more object-oriented. - Helps common operations else than API calls, such as content fetching and pagination control. - Provides `fsspec`_ filesystem implementation. @@ -101,7 +102,8 @@ efficient way. # Created file hello.png with mimeType image/png file3 = drive.CreateFile({'id': file2['id']}) - print('Downloading file %s from Google Drive' % file3['title']) # 'hello.png' + print('Downloading file %s from Google Drive' % file3['title']) + # 'hello.png' file3.GetContentFile('world.png') # Save Drive file as a local file # or download Google Docs files in an export format provided. @@ -122,7 +124,9 @@ File listing pagination made easy # Paginate file lists by specifying number of max results for file_list in drive.ListFile({'maxResults': 10}): - print('Received {} files from Files.list()'.format(len(file_list))) # <= 10 + print( + 'Received {} files from Files.list()'.format(len(file_list)) + ) # <= 10 for file1 in file_list: print('title: {}, id: {}'.format(file1['title'], file1['id'])) @@ -142,7 +146,8 @@ Install PyDrive2 with the required dependencies from pydrive2.fs import GDriveFileSystem - # replace `root` with ID of a drive or directory and give service account access to it + # replace `root` with ID of a drive or directory and give service account + # access to it fs = GDriveFileSystem("root", client_id=my_id, client_secret=my_secret) for root, dnames, fnames in fs.walk("root"): @@ -164,3 +169,7 @@ Thanks to all our contributors! :target: https://github.com/iterative/PyDrive2/graphs/contributors .. _`fsspec`: https://filesystem-spec.readthedocs.io/en/latest/ + +.. _google-api-python-client: + https://github.com/google/google-api-python-client +.. _Google Drive API V2: https://developers.google.com/drive/v2/web/about-sdk diff --git a/docs/filelist.rst b/docs/filelist.rst index 9089999..7854aaa 100644 --- a/docs/filelist.rst +++ b/docs/filelist.rst @@ -1,33 +1,39 @@ File listing made easy -============================= +====================== -*PyDrive* handles paginations and parses response as list of `GoogleDriveFile`_. +*PyDrive* handles paginations and parses response as list of +`GoogleDriveFile`_. Get all files which matches the query ------------------------------------- -Create `GoogleDriveFileList`_ instance with `parameters of Files.list()`_ as ``dict``. -Call `GetList()`_ and you will get all files that matches your query as a list of `GoogleDriveFile`_. -The syntax and possible option of the query ``q`` parameter can be found in `search for files` Google documentation. +Create `GoogleDriveFileList`_ instance with `parameters of Files.list()`_ as +``dict``. Call `GetList()`_ and you will get all files that matches your query +as a list of `GoogleDriveFile`_. The syntax and possible option of the query +``q`` parameter can be found in `search for files` Google documentation. .. code-block:: python from pydrive2.drive import GoogleDrive - drive = GoogleDrive(gauth) # Create GoogleDrive instance with authenticated GoogleAuth instance + # Create GoogleDrive instance with authenticated GoogleAuth instance + drive = GoogleDrive(gauth) # Auto-iterate through all files in the root folder. - file_list = drive.ListFile({'q': "'root' in parents and trashed=false"}).GetList() + file_list = drive.ListFile( + {'q': "'root' in parents and trashed=false"} + ).GetList() for file1 in file_list: - print('title: %s, id: %s' % (file1['title'], file1['id'])) + print('title: %s, id: %s' % (file1['title'], file1['id'])) -You can update metadata or content of these `GoogleDriveFile`_ instances if you need it. +You can update metadata or content of these `GoogleDriveFile`_ instances if you +need it. Paginate and iterate through files ---------------------------------- -*PyDrive* provides Pythonic way of paginating and iterating through list of files. -Here is an example how to do this, ``maxResults`` below defines how many +*PyDrive* provides Pythonic way of paginating and iterating through list of +files. Here is an example how to do this, ``maxResults`` below defines how many files it retrieves at once and we wrap it into a ``for`` loop to iterate: Sample code continues from above: @@ -36,13 +42,15 @@ Sample code continues from above: # Paginate file lists by specifying number of max results for file_list in drive.ListFile({'q': 'trashed=true', 'maxResults': 10}): - print('Received %s files from Files.list()' % len(file_list)) # <= 10 - for file1 in file_list: - print('title: %s, id: %s' % (file1['title'], file1['id'])) + print('Received %s files from Files.list()' % len(file_list)) # <= 10 + for file1 in file_list: + print('title: %s, id: %s' % (file1['title'], file1['id'])) .. _`GoogleDriveFile`: /PyDrive2/pydrive2/#pydrive2.files.GoogleDriveFile -.. _`GoogleDriveFileList`: /PyDrive2/pydrive2/#pydrive2.files.GoogleDriveFileList -.. _`parameters of Files.list()`: https://developers.google.com/drive/v2/reference/files/list#request +.. _`GoogleDriveFileList`: + /PyDrive2/pydrive2/#pydrive2.files.GoogleDriveFileList +.. _`parameters of Files.list()`: + https://developers.google.com/drive/v2/reference/files/list#request .. _`GetList()`: /PyDrive2/pydrive2/#pydrive2.apiattr.ApiResourceList.GetList .. _`search for files`: https://developers.google.com/drive/api/v2/search-files diff --git a/docs/filemanagement.rst b/docs/filemanagement.rst index 09c8d18..b38af6e 100644 --- a/docs/filemanagement.rst +++ b/docs/filemanagement.rst @@ -1,15 +1,16 @@ File management made easy ========================= -There are many methods to create and update file metadata and contents. -With *PyDrive*, you don't have to care about any of these different API methods. +There are many methods to create and update file metadata and contents. With +*PyDrive*, you don't have to care about any of these different API methods. Manipulate file metadata and contents from `GoogleDriveFile`_ object and call `Upload()`_. *PyDrive* will make the optimal API call for you. Upload a new file ----------------- -Here is a sample code to upload a file. ``gauth`` is an authenticated `GoogleAuth`_ object. +Here is a sample code to upload a file. ``gauth`` is an authenticated +`GoogleAuth`_ object. .. code-block:: python @@ -24,12 +25,17 @@ Here is a sample code to upload a file. ``gauth`` is an authenticated `GoogleAut print('title: %s, id: %s' % (file1['title'], file1['id'])) # title: Hello.txt, id: {{FILE_ID}} -Now, you will have a file 'Hello.txt' uploaded to your Google Drive. You can open it from web interface to check its content, 'Hello World!'. +Now, you will have a file 'Hello.txt' uploaded to your Google Drive. You can +open it from web interface to check its content, 'Hello World!'. -Note that `CreateFile()`_ will create `GoogleDriveFile`_ instance but not actually upload a file to Google Drive. You can initialize `GoogleDriveFile`_ object by itself. However, it is not recommended to do so in order to keep authentication consistent. +Note that `CreateFile()`_ will create `GoogleDriveFile`_ instance but not +actually upload a file to Google Drive. You can initialize `GoogleDriveFile`_ +object by itself. However, it is not recommended to do so in order to keep +authentication consistent. Delete, Trash and un-Trash files -------------------------------- + You may want to delete, trash, or un-trash a file. To do this use ``Delete()``, ``Trash()`` or ``UnTrash()`` on a GoogleDriveFile object. @@ -49,8 +55,9 @@ You may want to delete, trash, or un-trash a file. To do this use ``Delete()``, Update file metadata -------------------- -You can manipulate file metadata from a `GoogleDriveFile`_ object just as you manipulate a ``dict``. -The format of file metadata can be found in the Google Drive API documentation: `Files resource`_. +You can manipulate file metadata from a `GoogleDriveFile`_ object just as you +manipulate a ``dict``. The format of file metadata can be found in the Google +Drive API documentation: `Files resource`_. Sample code continues from `Upload a new file`_: @@ -90,7 +97,8 @@ PyDrive gives you access to the metadata of an object through file1 = drive.CreateFile({'id': ''}) - # Fetches all basic metadata fields, including file size, last modified etc. + # Fetches all basic metadata fields, including file size, last modified + # etc. file1.FetchMetadata() # Fetches all metadata available. @@ -106,6 +114,7 @@ For more information on available metadata fields have a look at the Insert permissions __________________ + Insert, retrieving or deleting permissions is illustrated by making a file readable to all who have a link to the file. @@ -126,7 +135,7 @@ Note: ``InsertPermission()`` calls ``GetPermissions()`` after successfully inserting the permission. You can find more information on the permitted fields of a permission -`here `_. +`here`_. This file is now shared and anyone with the link can view it. But what if you want to check whether a file is already shared? @@ -159,9 +168,10 @@ For the more advanced user: ``GetPermissions()`` is a shorthand for: Remove a Permission ___________________ + *PyDrive* allows you to remove a specific permission using the -``DeletePermission(permission_id)`` function. This function allows you to delete -one permission at a time by providing the permission's ID. +``DeletePermission(permission_id)`` function. This function allows you to +delete one permission at a time by providing the permission's ID. .. code-block:: python @@ -193,7 +203,7 @@ a list of `GoogleDriveFile`_ instances. files = drive.ListFile(query).GetList() List revisions -________________ +______________ Revisions can be fetched using the ``GetRevisions()`` function of a ``GoogleDriveFile``, and can be used like so: @@ -220,7 +230,10 @@ Sample code continues from `Download file metadata from file ID`_: .. code-block:: python - file4 = drive.CreateFile({'title':'appdata.json', 'mimeType':'application/json'}) + file4 = drive.CreateFile({ + 'title': 'appdata.json', + 'mimeType': 'application/json', + }) file4.SetContentString('{"firstname": "John", "lastname": "Smith"}') file4.Upload() # Upload file. file4.SetContentString('{"firstname": "Claudio", "lastname": "Afshar"}') @@ -246,10 +259,10 @@ like so: content_string = file4.GetContentString(encoding='ISO-8859-1') Upload data as bytes in memory buffer --------------------------------------- +------------------------------------- -Data can be kept as bytes in an in-memory buffer when we use the ``io`` module’s -Byte IO operations, we can upload files that reside in memory, for +Data can be kept as bytes in an in-memory buffer when we use the ``io`` +module’s Byte IO operations, we can upload files that reside in memory, for example we have a base64 image, we can decode the string and upload it to drive without the need to save as a file and use `SetContentFile(filename)`_ @@ -328,22 +341,32 @@ repository for an example. Abusive files ------------- -Files identified as `abusive`_ (malware, etc.) are only downloadable by the owner. -If you see a -'This file has been identified as malware or spam and cannot be downloaded' -error, set 'acknowledge_abuse=True' parameter in `GetContentFile()`. By using -it you indicate that you acknowledge the risks of downloading potential malware. +Files identified as `abusive`_ (malware, etc.) are only downloadable by the +owner. If you see a 'This file has been identified as malware or spam and +cannot be downloaded' error, set 'acknowledge_abuse=True' parameter in +`GetContentFile()`. By using it you indicate that you acknowledge the risks of +downloading potential malware. .. _`GoogleDriveFile`: /PyDrive2/pydrive2/#pydrive2.files.GoogleDriveFile .. _`Upload()`: /PyDrive2/pydrive2/#pydrive2.files.GoogleDriveFile.Upload .. _`GoogleAuth`: /PyDrive2/pydrive2/#pydrive2.auth.GoogleAuth .. _`CreateFile()`: /PyDrive2/pydrive2/#pydrive2.drive.GoogleDrive.CreateFile -.. _`Files resource`: https://developers.google.com/drive/v2/reference/files#resource-representations -.. _`SetContentFile(filename)`: /PyDrive2/pydrive2/#pydrive2.files.GoogleDriveFile.SetContentFile -.. _`SetContentString(content)`: /PyDrive2/pydrive2/#pydrive2.files.GoogleDriveFile.SetContentString -.. _`GetContentFile(filename)`: /PyDrive2/pydrive2/#pydrive2.files.GoogleDriveFile.GetContentFile -.. _`GetContentString()`: /PyDrive2/pydrive2/#pydrive2.files.GoogleDriveFile.GetContentString -.. _`official documentation`: https://developers.google.com/drive/v2/reference/files#resource-representations +.. _`Files resource`: + https://developers.google.com/drive/v2/reference/files#resource-representations +.. _`SetContentFile(filename)`: + /PyDrive2/pydrive2/#pydrive2.files.GoogleDriveFile.SetContentFile +.. _`SetContentString(content)`: + /PyDrive2/pydrive2/#pydrive2.files.GoogleDriveFile.SetContentString +.. _`GetContentFile(filename)`: + /PyDrive2/pydrive2/#pydrive2.files.GoogleDriveFile.GetContentFile +.. _`GetContentString()`: + /PyDrive2/pydrive2/#pydrive2.files.GoogleDriveFile.GetContentString +.. _`official documentation`: + https://developers.google.com/drive/v2/reference/files#resource-representations .. _`known`: https://productforums.google.com/forum/#!topic/docs/BJLimQDGtjQ .. _`abusive`: https://support.google.com/docs/answer/148505 -.. _`query_parameters`: https://developers.google.com/drive/api/guides/search-files#examples +.. _`query_parameters`: + https://developers.google.com/drive/api/guides/search-files#examples + +.. _here: + https://developers.google.com/drive/v2/reference/permissions/insert#request-body diff --git a/docs/fsspec.rst b/docs/fsspec.rst index 4714102..0882b1a 100644 --- a/docs/fsspec.rst +++ b/docs/fsspec.rst @@ -82,7 +82,8 @@ Using json keyfile path: from pydrive2.fs import GDriveFileSystem fs = GDriveFileSystem( - # replace with ID of a drive or directory and give service account access to it + # replace with ID of a drive or directory and give service account + # access to it "root", use_service_account=True, client_json_file_path="/path/to/keyfile.json", @@ -95,7 +96,8 @@ Using json keyfile string: from pydrive2.fs import GDriveFileSystem fs = GDriveFileSystem( - # replace with ID of a drive or directory and give service account access to it + # replace with ID of a drive or directory and give service account + # access to it "root", use_service_account=True, client_json=json_string, @@ -106,19 +108,23 @@ Use `client_user_email` if you are using `delegation of authority`_. Additional parameters --------------------- -:trash_only (bool): Move files to trash instead of deleting. -:acknowledge_abuse (bool): Acknowledging the risk and download file identified as abusive. See `Abusive files`_ for more info. +:trash_only (bool): + Move files to trash instead of deleting. +:acknowledge_abuse (bool): + Acknowledging the risk and download file identified as abusive. See + `Abusive files`_ for more info. Using filesystem ---------------- .. code-block:: python - # replace `root` with ID of a drive or directory and give service account access to it + # replace `root` with ID of a drive or directory and give service account + # access to it for root, dnames, fnames in fs.walk("root"): for dname in dnames: print(f"dir: {root}/{dname}") - + for fname in fnames: print(f"file: {root}/{fname}") @@ -128,6 +134,8 @@ about and manipulating files, refer to fsspec docs on .. _`fsspec`: https://filesystem-spec.readthedocs.io/en/latest/ .. _`GDriveFileSystem`: /PyDrive2/pydrive2/#pydrive2.fs.GDriveFileSystem -.. _`delegation of authority`: https://developers.google.com/admin-sdk/directory/v1/guides/delegation +.. _`delegation of authority`: + https://developers.google.com/admin-sdk/directory/v1/guides/delegation .. _`Abusive files`: /PyDrive2/filemanagement/index.html#abusive-files -.. _`how to use a filesystem`: https://filesystem-spec.readthedocs.io/en/latest/usage.html#use-a-file-system +.. _`how to use a filesystem`: + https://filesystem-spec.readthedocs.io/en/latest/usage.html#use-a-file-system diff --git a/docs/index.rst b/docs/index.rst index ddb667f..d576c9e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,15 +6,16 @@ Welcome to PyDrive2's documentation! ==================================== -PyDrive2 is a wrapper library of `google-api-python-client`_ that simplifies many common Google Drive API tasks. +PyDrive2 is a wrapper library of `google-api-python-client`_ that simplifies +many common Google Drive API tasks. Project Info ============ -- Package: `https://pypi.python.org/pypi/PyDrive2 `_ -- Documentation: `https://docs.iterative.ai/PyDrive2 `_ -- Source: `https://github.com/iterative/PyDrive2 `_ -- Changelog: `https://github.com/iterative/PyDrive2/releases `_ +- Package: https://pypi.python.org/pypi/PyDrive2 +- Documentation: https://docs.iterative.ai/PyDrive2 +- Source: https://github.com/iterative/PyDrive2 +- Changelog: https://github.com/iterative/PyDrive2/releases How to install ============== @@ -32,7 +33,8 @@ To install the current development version from GitHub, use: $ pip install git+https://github.com/iterative/PyDrive2.git#egg=PyDrive2 -.. _`google-api-python-client`: https://github.com/google/google-api-python-client +.. _`google-api-python-client`: + https://github.com/google/google-api-python-client Table of Contents ================= diff --git a/docs/oauth.rst b/docs/oauth.rst index eba60dc..160f8a4 100644 --- a/docs/oauth.rst +++ b/docs/oauth.rst @@ -15,8 +15,8 @@ OAuth2.0 is complex and difficult to start with. To make it more simple, # Create local webserver and auto handles authentication. gauth.LocalWebserverAuth() -To make this code work, you need to download the application configurations file -from APIs Console. Take a look at quickstart_ for detailed instructions. +To make this code work, you need to download the application configurations +file from APIs Console. Take a look at quickstart_ for detailed instructions. `LocalWebserverAuth()`_ is a built-in method of `GoogleAuth`_ which sets up local webserver to automatically receive authentication code from user and @@ -26,16 +26,18 @@ callback port on the machine running PyDrive2. For unattended jobs, use a service account or previously saved user credentials with a refresh token. .. _quickstart: /PyDrive2/quickstart/#authentication -.. _`LocalWebserverAuth()`: /PyDrive2/pydrive2/#pydrive2.auth.GoogleAuth.LocalWebserverAuth +.. _`LocalWebserverAuth()`: + /PyDrive2/pydrive2/#pydrive2.auth.GoogleAuth.LocalWebserverAuth .. _`GoogleAuth`: /PyDrive2/pydrive2/#pydrive2.auth.GoogleAuth -.. _`CommandLineAuth()`: /PyDrive2/pydrive2/#pydrive2.auth.GoogleAuth.CommandLineAuth +.. _`CommandLineAuth()`: + /PyDrive2/pydrive2/#pydrive2.auth.GoogleAuth.CommandLineAuth Automatic and custom authentication with *settings.yaml* -------------------------------------------------------- Read this section if you need a custom authentication flow, **such as silent -authentication on a remote machine**. For an example of such a setup have a look -at `Sample settings.yaml`_. +authentication on a remote machine**. For an example of such a setup have a +look at `Sample settings.yaml`_. OAuth is complicated and it requires a lot of settings. By default, when you don't provide any settings, *PyDrive* will automatically set default @@ -83,29 +85,91 @@ These are all the possible fields of a *settings.yaml* file: Fields explained: -:client_config_backend (str): From where to read client configuration(API application settings such as client_id and client_secrets) from. Valid values are 'file', 'settings' and 'service'. **Default**: 'file'. **Required**: No. -:client_config_file (str): When *client_config_backend* is 'file', path to the file containing client configuration. **Default**: 'client_secrets.json'. **Required**: No. -:client_config (dict): Place holding dictionary for client configuration when *client_config_backend* is 'settings'. **Required**: Yes, only if *client_config_backend* is 'settings' and not using *service_config* -:client_config['client_id'] (str): Client ID of the application. **Required**: Yes, only if *client_config_backend* is 'settings' -:client_config['client_secret'] (str): Client secret of the application. **Required**: Yes, only if *client_config_backend* is 'settings' -:client_config['auth_uri'] (str): The authorization server endpoint URI. **Default**: 'https://accounts.google.com/o/oauth2/auth'. **Required**: No. -:client_config['token_uri'] (str): The token server endpoint URI. **Default**: 'https://accounts.google.com/o/oauth2/token'. **Required**: No. -:client_config['redirect_uri'] (str): Redirection endpoint URI. **Default**: 'http://localhost'. **Required**: No. -:client_config['revoke_uri'] (str): Revoke endpoint URI. **Default**: None. **Required**: No. -:service_config (dict): Place holding dictionary for client configuration when *client_config_backend* is 'service' or 'settings' and using service account. **Required**: Yes, only if *client_config_backend* is 'service' or 'settings' and not using *client_config* -:service_config['client_user_email'] (str): User email that authority was delegated_ to. **Required**: No. -:service_config['client_json_file_path'] (str): Path to service account `.json` key file. **Required**: No. -:service_config['client_json_dict'] (dict): Service account `.json` key file loaded into a dictionary. **Required**: No. -:service_config['client_json'] (str): Service account `.json` key file loaded into a string. **Required**: No. -:save_credentials (bool): Save OAuth user credentials after signing in. **Default**: False. **Required**: No. -:save_credentials_backend (str): Backend to save credentials to. 'file' and 'dictionary' are the only valid values for now. **Default**: 'file'. **Required**: No. -:save_credentials_file (str): Destination of credentials file. **Required**: Yes, only if *save_credentials_backend* is 'file'. -:save_credentials_dict (dict): Dict to use for storing credentials. **Required**: Yes, only if *save_credentials_backend* is 'dictionary'. -:save_credentials_key (str): Key within the *save_credentials_dict* to store the credentials in. **Required**: Yes, only if *save_credentials_backend* is 'dictionary'. -:get_refresh_token (bool): True if you want to retrieve refresh token along with access token. **Default**: False. **Required**: No. -:oauth_scope (list of str): OAuth scope to authenticate. **Default**: ['https://www.googleapis.com/auth/drive']. **Required**: No. - -.. _delegated: https://developers.google.com/admin-sdk/directory/v1/guides/delegation +:client_config_backend (str): + From where to read client configuration(API application settings such as + client_id and client_secrets) from. Valid values are 'file', 'settings' and + 'service'. + **Default**: 'file'. + **Required**: No. +:client_config_file (str): + When *client_config_backend* is 'file', path to the file containing client + configuration. + **Default**: 'client_secrets.json'. + **Required**: No. +:client_config (dict): + Place holding dictionary for client configuration when + *client_config_backend* is 'settings'. + **Required**: Yes, only if *client_config_backend* is 'settings' and not + using *service_config* +:client_config['client_id'] (str): + Client ID of the application. + **Required**: Yes, only if *client_config_backend* is 'settings' +:client_config['client_secret'] (str): + Client secret of the application. + **Required**: Yes, only if *client_config_backend* is 'settings' +:client_config['auth_uri'] (str): + The authorization server endpoint URI. + **Default**: 'https://accounts.google.com/o/oauth2/auth'. + **Required**: No. +:client_config['token_uri'] (str): + The token server endpoint URI. + **Default**: 'https://accounts.google.com/o/oauth2/token'. + **Required**: No. +:client_config['redirect_uri'] (str): + Redirection endpoint URI. + **Default**: 'http://localhost'. + **Required**: No. +:client_config['revoke_uri'] (str): + Revoke endpoint URI. + **Default**: None. + **Required**: No. +:service_config (dict): + Place holding dictionary for client configuration when + *client_config_backend* is 'service' or 'settings' and using service + account. + **Required**: Yes, only if *client_config_backend* is 'service' or + 'settings' and not using *client_config* +:service_config['client_user_email'] (str): + User email that authority was delegated_ to. + **Required**: No. +:service_config['client_json_file_path'] (str): + Path to service account `.json` key file. + **Required**: No. +:service_config['client_json_dict'] (dict): + Service account `.json` key file loaded into a dictionary. + **Required**: No. +:service_config['client_json'] (str): + Service account `.json` key file loaded into a string. + **Required**: No. +:save_credentials (bool): + Save OAuth user credentials after signing in. + **Default**: False. + **Required**: No. +:save_credentials_backend (str): + Backend to save credentials to. 'file' and 'dictionary' are the only valid + values for now. + **Default**: 'file'. + **Required**: No. +:save_credentials_file (str): + Destination of credentials file. + **Required**: Yes, only if *save_credentials_backend* is 'file'. +:save_credentials_dict (dict): + Dict to use for storing credentials. + **Required**: Yes, only if *save_credentials_backend* is 'dictionary'. +:save_credentials_key (str): + Key within the *save_credentials_dict* to store the credentials in. + **Required**: Yes, only if *save_credentials_backend* is 'dictionary'. +:get_refresh_token (bool): + True if you want to retrieve refresh token along with access token. + **Default**: False. + **Required**: No. +:oauth_scope (list of str): + OAuth scope to authenticate. + **Default**: ['https://www.googleapis.com/auth/drive']. + **Required**: No. + +.. _delegated: + https://developers.google.com/admin-sdk/directory/v1/guides/delegation Upgrading from PyDrive2 1.x ___________________________ @@ -149,7 +213,9 @@ want to integrate your existing website with Drive API. In that case, you can customize an authentication flow as follows: 1. Get authentication Url from `GetAuthUrl()`_. -2. Direct the user to the authorization URL. Receive the authorization code at the redirect URI registered for your application and verify the OAuth state in the callback. +2. Direct the user to the authorization URL. Receive the authorization code at + the redirect URI registered for your application and verify the OAuth state + in the callback. 3. Call `Auth(code)`_ with the authentication code you retrieved from step 2. Your *settings.yaml* will work for your customized authentication flow, too. @@ -160,9 +226,12 @@ Here is a sample code for your customized authentication flow from pydrive2.auth import GoogleAuth - gauth = GoogleAuth() # Configure your registered redirect URI in settings.yaml - auth_url = gauth.GetAuthUrl() # Create authentication url user needs to visit - code = ReceiveVerifiedCallback(auth_url) # Your application handles the redirect + # Configure your registered redirect URI in settings.yaml + gauth = GoogleAuth() + # Create authentication url user needs to visit + auth_url = gauth.GetAuthUrl() + # Your application handles the redirect + code = ReceiveVerifiedCallback(auth_url) gauth.Auth(code) # Authorize and build service from the code .. _`GetAuthUrl()`: /PyDrive2/pydrive2/#pydrive2.auth.GoogleAuth.GetAuthUrl @@ -170,10 +239,10 @@ Here is a sample code for your customized authentication flow Authentication with a service account --------------------------------------- +------------------------------------- -A `Service account`_ is a special type of Google account intended to represent a -non-human user that needs to authenticate and be authorized to access data in +A `Service account`_ is a special type of Google account intended to represent +a non-human user that needs to authenticate and be authorized to access data in Google APIs. Typically, service accounts are used in scenarios such as: @@ -214,4 +283,5 @@ If we use OAuth client ID we need to do one manual login into the account with return gauth -.. _`Service account`: https://developers.google.com/workspace/guides/create-credentials#service-account +.. _`Service account`: + https://developers.google.com/workspace/guides/create-credentials#service-account diff --git a/docs/pydrive2.rst b/docs/pydrive2.rst index bcbf0fc..762fbd1 100644 --- a/docs/pydrive2.rst +++ b/docs/pydrive2.rst @@ -42,7 +42,7 @@ pydrive2.settings module :show-inheritance: pydrive2.fs module ------------------------- +------------------ .. autoclass:: pydrive2.fs.GDriveFileSystem :show-inheritance: diff --git a/docs/quickstart.rst b/docs/quickstart.rst index a9a5886..37b32ca 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -1,24 +1,30 @@ Quickstart -============================= +========== Authentication -------------- -Drive API requires OAuth2.0 for authentication. *PyDrive2* makes your life much easier by handling complex authentication steps for you. + +Drive API requires OAuth2.0 for authentication. *PyDrive2* makes your life much +easier by handling complex authentication steps for you. 1. Go to `APIs Console`_ and make your own project. 2. Search for 'Google Drive API', select the entry, and click 'Enable'. -3. Select 'Credentials' from the left menu, click 'Create Credentials', select 'OAuth client ID'. -4. Now, the product name and consent screen need to be set -> click 'Configure consent screen' and follow the instructions. Once finished: +3. Select 'Credentials' from the left menu, click 'Create Credentials', select + 'OAuth client ID'. +4. Now, the product name and consent screen need to be set -> click 'Configure + consent screen' and follow the instructions. Once finished: - a. Select 'Application type' to be *Web application*. - b. Enter an appropriate name. - c. Input *http://localhost:8080/* for 'Authorized redirect URIs'. - d. Click 'Create'. + a. Select 'Application type' to be *Web application*. + b. Enter an appropriate name. + c. Input *http://localhost:8080/* for 'Authorized redirect URIs'. + d. Click 'Create'. -5. Click 'Download JSON' on the right side of Client ID to download **client_secret_.json**. +5. Click 'Download JSON' on the right side of Client ID to download + **client_secret_.json**. The downloaded file has all authentication information of your application. -**Rename the file to "client_secrets.json" and place it in your working directory.** +**Rename the file to "client_secrets.json" and place it in your working +directory.** Create *quickstart.py* file and copy and paste the following code. @@ -27,9 +33,12 @@ Create *quickstart.py* file and copy and paste the following code. from pydrive2.auth import GoogleAuth gauth = GoogleAuth() - gauth.LocalWebserverAuth() # Creates local webserver and auto handles authentication. + # Creates local webserver and auto handles authentication. + gauth.LocalWebserverAuth() -Run this code with *python quickstart.py* and you will see a web browser asking you for authentication. Click *Accept* and you are done with authentication. For more details, take a look at documentation: `OAuth made easy`_ +Run this code with *python quickstart.py* and you will see a web browser asking +you for authentication. Click *Accept* and you are done with authentication. +For more details, take a look at documentation: `OAuth made easy`_ .. _`APIs Console`: https://console.developers.google.com/iam-admin/projects .. _`OAuth made easy`: /PyDrive2/oauth/ @@ -37,8 +46,9 @@ Run this code with *python quickstart.py* and you will see a web browser asking Creating and Updating Files --------------------------- -There are many methods to create and update file metadata and contents. With *PyDrive2*, all you have to know is -`Upload()`_ method which makes optimal API call for you. Add the following code to your *quickstart.py* and run it. +There are many methods to create and update file metadata and contents. With +*PyDrive2*, all you have to know is `Upload()`_ method which makes optimal API +call for you. Add the following code to your *quickstart.py* and run it. .. code-block:: python @@ -46,12 +56,15 @@ There are many methods to create and update file metadata and contents. With *Py drive = GoogleDrive(gauth) - file1 = drive.CreateFile({'title': 'Hello.txt'}) # Create GoogleDriveFile instance with title 'Hello.txt'. - file1.SetContentString('Hello World!') # Set content of the file from given string. + # Create GoogleDriveFile instance with title 'Hello.txt'. + file1 = drive.CreateFile({'title': 'Hello.txt'}) + # Set content of the file from given string. + file1.SetContentString('Hello World!') file1.Upload() -This code will create a new file with title *Hello.txt* and its content *Hello World!*. You can see and open this - file from `Google Drive`_ if you want. For more details, take a look at documentation: `File management made easy`_ +This code will create a new file with title *Hello.txt* and its content *Hello +World!*. You can see and open this file from `Google Drive`_ if you want. For +more details, take a look at documentation: `File management made easy`_ .. _`Upload()`: /PyDrive2/pydrive2/#pydrive2.files.GoogleDriveFile.Upload .. _`Google Drive`: https://drive.google.com @@ -60,86 +73,110 @@ This code will create a new file with title *Hello.txt* and its content *Hello W Listing Files ------------- -*PyDrive2* handles paginations and parses response as list of `GoogleDriveFile`_. Let's get title and id of all the files in the root folder of Google Drive. Again, add the following code to *quickstart.py* and execute it. +*PyDrive2* handles paginations and parses response as list of +`GoogleDriveFile`_. Let's get title and id of all the files in the root folder +of Google Drive. Again, add the following code to *quickstart.py* and execute +it. .. code-block:: python # Auto-iterate through all files that matches this query - file_list = drive.ListFile({'q': "'root' in parents and trashed=false"}).GetList() + file_list = drive.ListFile( + {'q': "'root' in parents and trashed=false"} + ).GetList() for file1 in file_list: - print('title: %s, id: %s' % (file1['title'], file1['id'])) + print('title: %s, id: %s' % (file1['title'], file1['id'])) Creating a Folder ----------------- -GoogleDrive treats everything as a file and assigns different mimetypes for different file formats. A folder is thus - also a file with a special mimetype. The code below allows you to add a subfolder to an existing folder. +GoogleDrive treats everything as a file and assigns different mimetypes for +different file formats. A folder is thus also a file with a special mimetype. +The code below allows you to add a subfolder to an existing folder. .. code-block:: python def create_folder(parent_folder_id, subfolder_name): - newFolder = drive.CreateFile({'title': subfolder_name, "parents": [{"kind": "drive#fileLink", "id": \ - parent_folder_id}],"mimeType": "application/vnd.google-apps.folder"}) - newFolder.Upload() - return newFolder + newFolder = drive.CreateFile({ + 'title': subfolder_name, + "parents": [{"kind": "drive#fileLink", "id": parent_folder_id}], + "mimeType": "application/vnd.google-apps.folder", + }) + newFolder.Upload() + return newFolder Return File ID via File Title ----------------------------- A common task is providing the Google Drive API with a file id. -``get_id_of_title`` demonstrates a simple workflow to return the id of a file handle by searching the file titles in a -given directory. The function takes two arguments, ``title`` and ``parent_directory_id``. ``title`` is a string that -will be compared against file titles included in a directory identified by the ``parent_directory_id``. +``get_id_of_title`` demonstrates a simple workflow to return the id of a file +handle by searching the file titles in a given directory. The function takes +two arguments, ``title`` and ``parent_directory_id``. ``title`` is a string +that will be compared against file titles included in a directory identified by +the ``parent_directory_id``. .. code-block:: python - def get_id_of_title(title,parent_directory_id): - foldered_list=drive.ListFile({'q': "'"+parent_directory_id+"' in parents and trashed=false"}).GetList() - for file in foldered_list: - if(file['title']==title): - return file['id'] - return None + def get_id_of_title(title, parent_directory_id): + foldered_list = drive.ListFile({ + 'q': "'" + parent_directory_id + "' in parents and trashed=false" + }).GetList() + for file in foldered_list: + if file['title'] == title: + return file['id'] + return None Browse Folders -------------- -This returns a json output of the data in a directory with some important attributes like size, title, parent_id. + +This returns a json output of the data in a directory with some important +attributes like size, title, parent_id. .. code-block:: python - browsed=[] - def folder_browser(folder_list,parent_id): - for element in folder_list: - if type(element) is dict: - print (element['title']) - else: - print (element) - print("Enter Name of Folder You Want to Use\nEnter '/' to use current folder\nEnter ':' to create New Folder and - use that" ) - inp=input() - if inp=='/': - return parent_id - elif inp==':': - print("Enter Name of Folder You Want to Create") - inp=input() - newfolder=create_folder(parent_id,inp) - if not os.path.exists(HOME_DIRECTORY+ROOT_FOLDER_NAME+os.path.sep+USERNAME): - os.makedirs(HOME_DIRECTORY+ROOT_FOLDER_NAME+os.path.sep+USERNAME) - return newfolder['id'] - - else: - folder_selected=inp + browsed = [] + + def folder_browser(folder_list, parent_id): for element in folder_list: - if type(element) is dict: - if element["title"]==folder_selected: - struc=element["list"] - browsed.append(folder_selected) - print("Inside "+folder_selected) - return folder_browser(struc,element['id']) + if type(element) is dict: + print(element['title']) + else: + print(element) + print( + "Enter Name of Folder You Want to Use\n" + "Enter '/' to use current folder\n" + "Enter ':' to create New Folder and use that" + ) + inp = input() + if inp == '/': + return parent_id + elif inp == ':': + print("Enter Name of Folder You Want to Create") + inp = input() + newfolder = create_folder(parent_id, inp) + if not os.path.exists( + HOME_DIRECTORY + ROOT_FOLDER_NAME + os.path.sep + USERNAME + ): + os.makedirs( + HOME_DIRECTORY + ROOT_FOLDER_NAME + os.path.sep + USERNAME + ) + return newfolder['id'] + else: + folder_selected = inp + for element in folder_list: + if type(element) is dict: + if element["title"] == folder_selected: + struc = element["list"] + browsed.append(folder_selected) + print("Inside " + folder_selected) + return folder_browser(struc, element['id']) here ``folder_list`` is the list of folders that is present -You will see title and id of all the files and folders in root folder of your Google Drive. For more details, refer to the documentation: `File listing made easy`_ +You will see title and id of all the files and folders in root folder of your +Google Drive. For more details, refer to the documentation: +`File listing made easy`_ .. _`GoogleDriveFile`: /PyDrive2/pydrive2/#pydrive2.files.GoogleDriveFile .. _`File listing made easy`: /PyDrive2/filelist/ diff --git a/pydrive2/auth.py b/pydrive2/auth.py index fc02402..20199be 100644 --- a/pydrive2/auth.py +++ b/pydrive2/auth.py @@ -1,5 +1,3 @@ -from __future__ import annotations - import errno import json import os diff --git a/pydrive2/test/README.rst b/pydrive2/test/README.rst index 4fa2ef1..8dd7be5 100644 --- a/pydrive2/test/README.rst +++ b/pydrive2/test/README.rst @@ -13,66 +13,67 @@ those tests locally, configure a service account as described below. 1. Copy settings files to the :code:`pydrive2/test/settings/local` directory: -:: + :: - cd pydrive2/test/settings && cp *.yaml local + cd pydrive2/test/settings && cp *.yaml local 2. Setup a Google service account for your Google Cloud Project: - - Sign into the `Google API Console - `_ - - Select or `Create a new - `_ - project. - - `Enable the Drive API - `_ from the **APIs & - Services** **Dashboard** (left sidebar), click on **+ ENABLE APIS AND - SERVICES**. Find and select the "Google Drive API" in the API Library, and - click on the **ENABLE** button. - - Go back to **IAM & Admin** in the left - sidebar, and select **Service Accounts**. Click **+ CREATE SERVICE - ACCOUNT**, on the next screen, enter **Service account name** e.g. "PyDrive - tests", and click **Create**. Select **Continue** at the next **Service - account permissions** page, click at **+ CREATE KEY**, select **JSON** and - **Create**. Save generated :code:`.json` key file at your local disk. - - Copy downloaded :code:`json` file to :code:`/tmp/pydrive2/credentials.json` - directory. - -3. Optional. If you would like to use your own an OAuth client ID follow the steps: - - Under `Google API Console `_ select - **APIs & Services** from the left sidebar, and select **OAuth consent screen**. - Chose a **User Type** and click **CREATE**. On the next screen, enter an - **Application name** e.g. "PyDrive tests", and click the **Save** (scroll to - bottom). - - From the left sidebar, select **Credentials**, and click the - **Create credentials** dropdown to select **OAuth client ID**. Chose **Other** - and click **Create** to proceed with a default client name. At **Credentials** - screen find a list of your **OAuth 2.0 Client IDs**, click download icon in - front of your OAuth client id created previously. You should be prompted to - download :code:`client_secret_xxx_.json` file. - - Copy downloaded :code:`.json` file into :code:`pydrive2/test` directory - and rename to :code:`client_secrets.json`. - - Replace {{ }} sections - in :code:`pydrive2/test/settings/local/test_oauth_test_02.yaml` with the - relevant values of :code:`client_id` and :code:`client_secret` from your - **client_secrets.json** file. -4. Setup virtual environment (recommended optional step): + - Sign into the `Google API Console`_ + - Select or `Create a new`_ project. + - `Enable the Drive API`_ from the **APIs & Services** **Dashboard** (left + sidebar), click on **+ ENABLE APIS AND SERVICES**. Find and select the + "Google Drive API" in the API Library, and click on the **ENABLE** button. + - Go back to **IAM & Admin** in the left sidebar, and select + **Service Accounts**. Click **+ CREATE SERVICE ACCOUNT**, on the next + screen, enter **Service account name** e.g. "PyDrive tests", and click + **Create**. Select **Continue** at the next + **Service account permissions** page, click at **+ CREATE KEY**, select + **JSON** and **Create**. Save generated :code:`.json` key file at your + local disk. + - Copy downloaded :code:`json` file to + :code:`/tmp/pydrive2/credentials.json` directory. + +3. Optional. If you would like to use your own an OAuth client ID follow the + steps: + + - Under `Google API Console`_ select **APIs & Services** from the left + sidebar, and select **OAuth consent screen**. Chose a **User Type** and + click **CREATE**. On the next screen, enter an **Application name** e.g. + "PyDrive tests", and click the **Save** (scroll to bottom). + - From the left sidebar, select **Credentials**, and click the + **Create credentials** dropdown to select **OAuth client ID**. Chose + **Other** and click **Create** to proceed with a default client name. + At **Credentials** screen find a list of your **OAuth 2.0 Client IDs**, + click download icon in front of your OAuth client id created previously. + You should be prompted to download :code:`client_secret_xxx_.json` file. + - Copy downloaded :code:`.json` file into :code:`pydrive2/test` directory + and rename to :code:`client_secrets.json`. + - Replace {{ }} sections in + :code:`pydrive2/test/settings/local/test_oauth_test_02.yaml` with the + relevant values of :code:`client_id` and :code:`client_secret` from your + **client_secrets.json** file. -:: +4. Setup virtual environment (recommended optional step): + :: - virtualenv -p python .env - source .env/bin/activate + virtualenv -p python .env + source .env/bin/activate 5. Install :code:`tests` deps from the root directory of the project: -:: + :: - pip install -e .[tests,fsspec] + pip install -e .[tests,fsspec] +6. Run tests: -5. Run tests: + :: -:: + py.test -v -s - py.test -v -s +.. _Google API Console: https://console.developers.google.com +.. _Create a new: + https://cloud.google.com/resource-manager/docs/creating-managing-projects#creating_a_project +.. _Enable the Drive API: https://developers.google.com/drive/api/v2/about-sdk From 050745f6a2f28cd750e264a02333b664d21566df Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Thu, 19 Nov 2026 22:31:00 -0800 Subject: [PATCH 06/15] Remove redundant credential module docstring --- pydrive2/_credentials.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pydrive2/_credentials.py b/pydrive2/_credentials.py index 2079367..1c07426 100644 --- a/pydrive2/_credentials.py +++ b/pydrive2/_credentials.py @@ -1,5 +1,3 @@ -"""Credential persistence for GoogleAuth's file and dictionary backends.""" - import json import os import tempfile From 10d072fdcd72c8f84d19c00b061ebf3da453ea63 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Fri, 20 Nov 2026 00:40:50 -0800 Subject: [PATCH 07/15] Keep user credential persistence in the auth module --- pydrive2/_credentials.py | 103 ---------------- pydrive2/auth.py | 239 ++++++++++++++++++++++-------------- pydrive2/test/test_oauth.py | 8 +- 3 files changed, 151 insertions(+), 199 deletions(-) delete mode 100644 pydrive2/_credentials.py diff --git a/pydrive2/_credentials.py b/pydrive2/_credentials.py deleted file mode 100644 index 1c07426..0000000 --- a/pydrive2/_credentials.py +++ /dev/null @@ -1,103 +0,0 @@ -import json -import os -import tempfile -import threading -from pathlib import Path - -from google.oauth2.credentials import Credentials - - -class InvalidCredentialsError(IOError): - """Error trying to read or write credentials.""" - - -class UserCredentials(Credentials): - storage = None - - def refresh(self, request): - storage = self.storage - if storage is None: - return super().refresh(request) - - # AuthorizedHttp can refresh during a request, including after a 401. - # Persist rotated tokens on that path as well as on explicit Refresh(). - previous_token = self.token - with storage.lock: - if self.token == previous_token or not self.valid: - super().refresh(request) - storage.put(self) - - -def load_credentials(serialized, storage): - if not serialized: - return None - try: - info = json.loads(serialized) - if ( - not isinstance(info, dict) - or info.get("type", "authorized_user") != "authorized_user" - or "_class" in info - or "access_token" in info - or not (info.get("token") or info.get("refresh_token")) - ): - raise ValueError("Expected google-auth user credentials") - # Online-only logins have an access token but no refresh token. - info.setdefault("refresh_token", None) - cred = UserCredentials.from_authorized_user_info(info) - except (AttributeError, TypeError, ValueError) as exc: - raise InvalidCredentialsError( - "Select a fresh user credential file or dictionary key and sign " - "in again with PyDrive2 2.x." - ) from exc - cred.storage = storage - return cred - - -class FileStorage: - def __init__(self, filename): - self.path = Path(filename) - self.lock = threading.RLock() - - def _check_path(self): - if self.path.is_symlink(): - raise InvalidCredentialsError( - "Credentials file cannot be symbolic link" - ) - - def get(self): - with self.lock: - self._check_path() - flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) - try: - fd = os.open(self.path, flags) - except FileNotFoundError: - return None - with os.fdopen(fd, encoding="utf-8") as stream: - return load_credentials(stream.read(), self) - - def put(self, credentials): - with self.lock: - self._check_path() - self.path.parent.mkdir(parents=True, exist_ok=True) - fd, name = tempfile.mkstemp(dir=self.path.parent) - try: - with os.fdopen(fd, "w", encoding="utf-8") as stream: - stream.write(credentials.to_json()) - os.replace(name, self.path) - finally: - Path(name).unlink(missing_ok=True) - - -class DictionaryStorage: - def __init__(self, dictionary, key): - self.dictionary = dictionary - self.key = key - self.lock = threading.RLock() - - def get(self): - with self.lock: - return load_credentials(self.dictionary.get(self.key), self) - - def put(self, credentials): - with self.lock: - self.dictionary[self.key] = credentials.to_json() diff --git a/pydrive2/auth.py b/pydrive2/auth.py index 20199be..c75de33 100644 --- a/pydrive2/auth.py +++ b/pydrive2/auth.py @@ -1,9 +1,11 @@ import errno import json import os +import tempfile import threading import webbrowser from functools import wraps +from pathlib import Path from typing import ClassVar import httplib2 @@ -17,12 +19,6 @@ from googleapiclient.discovery import build from oauthlib.oauth2 import AccessDeniedError, OAuth2Error -from ._credentials import ( - DictionaryStorage, - FileStorage, - InvalidCredentialsError, - UserCredentials, -) from .apiattr import ApiAttribute, ApiAttributeMixin from .settings import ( InvalidConfigError, @@ -44,10 +40,81 @@ class AuthenticationError(AuthError): """General authentication error.""" +class InvalidCredentialsError(IOError): + """Error trying to read or write credentials.""" + + class RefreshError(AuthError): """Access token refresh error.""" +class _UserCredentials(user_credentials.Credentials): + storage = None + + def refresh(self, request): + storage = self.storage + if storage is None: + return super().refresh(request) + + # AuthorizedHttp can refresh during a request, including after a 401. + # Persist rotated tokens on that path as well as on explicit Refresh(). + previous_token = self.token + with storage.lock: + if self.token == previous_token or not self.valid: + super().refresh(request) + storage.put(self) + + +class _FileStorage: + def __init__(self, filename): + self.path = Path(filename) + self.lock = threading.RLock() + + def _check_path(self): + if self.path.is_symlink(): + raise InvalidCredentialsError( + "Credentials file cannot be symbolic link" + ) + + def get(self): + with self.lock: + self._check_path() + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + try: + fd = os.open(self.path, flags) + except FileNotFoundError: + return None + with os.fdopen(fd, encoding="utf-8") as stream: + return stream.read() + + def put(self, credentials): + with self.lock: + self._check_path() + self.path.parent.mkdir(parents=True, exist_ok=True) + fd, name = tempfile.mkstemp(dir=self.path.parent) + try: + with os.fdopen(fd, "w", encoding="utf-8") as stream: + stream.write(credentials.to_json()) + os.replace(name, self.path) + finally: + Path(name).unlink(missing_ok=True) + + +class _DictionaryStorage: + def __init__(self, dictionary, key): + self.dictionary = dictionary + self.key = key + self.lock = threading.RLock() + + def get(self): + with self.lock: + return self.dictionary.get(self.key) + + def put(self, credentials): + with self.lock: + self.dictionary[self.key] = credentials.to_json() + + def LoadAuth(decoratee): """Decorator to check if the auth is valid and loads auth if not.""" @@ -153,9 +220,8 @@ def __init__( self.settings = settings or self.DEFAULT_SETTINGS ValidateSettings(self.settings) - storages, default = self._InitializeStoragesFromSettings() - self._storages = storages - self._default_storage = default + self._configured_storage = self._storage_from_settings() + self._default_storage = self._configured_storage @property def access_token_expired(self): @@ -166,7 +232,7 @@ def access_token_expired(self): return self.credentials is None or not self.credentials.valid def _use_flow_credentials(self, credentials): - self.credentials = UserCredentials( + self.credentials = _UserCredentials( token=credentials.token, refresh_token=credentials.refresh_token, id_token=credentials.id_token, @@ -313,35 +379,52 @@ def ServiceAuth(self): if self.service is None: self.Authorize() - def _InitializeStoragesFromSettings(self): - result: dict[str, FileStorage | DictionaryStorage | None] = { - "file": None, - "dictionary": None, - } + def _storage_from_settings(self): backend = self.settings.get("save_credentials_backend") - save_credentials = self.settings.get("save_credentials") if backend == "file": - credentials_file = self.settings.get("save_credentials_file") - if credentials_file is None: + filename = self.settings.get("save_credentials_file") + if filename is None: raise InvalidConfigError( "Please specify credentials file to read" ) - result[backend] = FileStorage(credentials_file) - elif backend == "dictionary": - creds_dict = self.settings.get("save_credentials_dict") - if creds_dict is None: + return _FileStorage(filename) + if backend == "dictionary": + dictionary = self.settings.get("save_credentials_dict") + key = self.settings.get("save_credentials_key") + if dictionary is None: raise InvalidConfigError("Please specify credentials dict") - - creds_key = self.settings.get("save_credentials_key") - if creds_key is None: + if key is None: raise InvalidConfigError("Please specify credentials key") - - result[backend] = DictionaryStorage(creds_dict, creds_key) - elif save_credentials: + return _DictionaryStorage(dictionary, key) + if self.settings.get("save_credentials"): raise InvalidConfigError( f"Unknown save_credentials_backend: {backend}" ) - return result, result.get(backend) + return None + + def _credential_backend(self, backend): + if backend is None: + backend = self.settings.get("save_credentials_backend") + if backend is None: + raise InvalidConfigError("Please specify credential backend") + if backend not in ("file", "dictionary"): + raise InvalidConfigError("Unknown save_credentials_backend") + return backend + + def _storage_for(self, backend, filename=None): + if filename is not None: + return _FileStorage(filename) + if backend == "file": + if isinstance(self._configured_storage, _FileStorage): + return self._configured_storage + raise InvalidConfigError( + "Set save_credentials_file or pass an explicit credentials file" + ) + if isinstance(self._configured_storage, _DictionaryStorage): + return self._configured_storage + raise InvalidConfigError( + "Set save_credentials_dict and save_credentials_key" + ) def LoadCredentials(self, backend=None): """Loads credentials or create empty credentials if it doesn't exist. @@ -350,16 +433,10 @@ def LoadCredentials(self, backend=None): :type backend: str. :raises: InvalidConfigError """ - if backend is None: - backend = self.settings.get("save_credentials_backend") - if backend is None: - raise InvalidConfigError("Please specify credential backend") - if backend == "file": + if self._credential_backend(backend) == "file": self.LoadCredentialsFile() - elif backend == "dictionary": - self._LoadCredentialsDictionary() else: - raise InvalidConfigError("Unknown save_credentials_backend") + self._load_credentials(self._storage_for("dictionary")) def LoadCredentialsFile(self, credentials_file=None): """Loads credentials or create empty credentials if it doesn't exist. @@ -370,28 +447,34 @@ def LoadCredentialsFile(self, credentials_file=None): :type credentials_file: str. :raises: InvalidConfigError, InvalidCredentialsError """ - if credentials_file is None: - self._default_storage = self._storages["file"] - if self._default_storage is None: - raise InvalidConfigError( - "Backend `file` is not configured, specify " - "credentials file to read in the settings " - "file or pass an explicit value" - ) - else: - self._default_storage = FileStorage(credentials_file) - - self.credentials = self._default_storage.get() - - def _LoadCredentialsDictionary(self): - self._default_storage = self._storages["dictionary"] - if self._default_storage is None: - raise InvalidConfigError( - "Backend `dictionary` is not configured, specify " - "credentials dict and key to read in the settings file" - ) + self._load_credentials(self._storage_for("file", credentials_file)) - self.credentials = self._default_storage.get() + def _load_credentials(self, storage): + self._default_storage = storage + serialized = storage.get() + if not serialized: + self.credentials = None + return + try: + info = json.loads(serialized) + if ( + not isinstance(info, dict) + or info.get("type", "authorized_user") != "authorized_user" + or "_class" in info + or "access_token" in info + or not (info.get("token") or info.get("refresh_token")) + ): + raise ValueError("Expected google-auth user credentials") + # Online-only logins have an access token but no refresh token. + info.setdefault("refresh_token", None) + cred = _UserCredentials.from_authorized_user_info(info) + except (AttributeError, TypeError, ValueError) as exc: + raise InvalidCredentialsError( + "Select a fresh user credential file or dictionary key and sign " + "in again with PyDrive2 2.x." + ) from exc + cred.storage = storage + self.credentials = cred def SaveCredentials(self, backend=None): """Saves credentials according to specified backend. @@ -403,16 +486,10 @@ def SaveCredentials(self, backend=None): :type backend: str. :raises: InvalidConfigError """ - if backend is None: - backend = self.settings.get("save_credentials_backend") - if backend is None: - raise InvalidConfigError("Please specify credential backend") - if backend == "file": + if self._credential_backend(backend) == "file": self.SaveCredentialsFile() - elif backend == "dictionary": - self._SaveCredentialsDictionary() else: - raise InvalidConfigError("Unknown save_credentials_backend") + self._save_credentials("dictionary") def SaveCredentialsFile(self, credentials_file=None): """Saves credentials to the file in JSON format. @@ -421,38 +498,14 @@ def SaveCredentialsFile(self, credentials_file=None): :type credentials_file: str. :raises: InvalidConfigError, InvalidCredentialsError """ - if not isinstance(self.credentials, user_credentials.Credentials): - raise InvalidCredentialsError( - "Only signed-in user credentials can be saved" - ) + self._save_credentials("file", credentials_file) - if credentials_file is None: - storage = self._storages["file"] - if storage is None: - raise InvalidConfigError( - "Backend `file` is not configured, specify " - "credentials file to read in the settings " - "file or pass an explicit value" - ) - else: - storage = FileStorage(credentials_file) - - storage.put(self.credentials) - - def _SaveCredentialsDictionary(self): + def _save_credentials(self, backend, filename=None): if not isinstance(self.credentials, user_credentials.Credentials): raise InvalidCredentialsError( "Only signed-in user credentials can be saved" ) - - storage = self._storages["dictionary"] - if storage is None: - raise InvalidConfigError( - "Backend `dictionary` is not configured, specify " - "credentials dict and key to write in the settings file" - ) - - storage.put(self.credentials) + self._storage_for(backend, filename).put(self.credentials) def LoadClientConfig(self, backend=None): """Loads client configuration according to specified backend. diff --git a/pydrive2/test/test_oauth.py b/pydrive2/test/test_oauth.py index 125de27..5187f5d 100644 --- a/pydrive2/test/test_oauth.py +++ b/pydrive2/test/test_oauth.py @@ -6,7 +6,6 @@ import pytest from google.oauth2.credentials import Credentials -from pydrive2._credentials import FileStorage from pydrive2.auth import AuthenticationError, GoogleAuth from pydrive2.test.test_util import ( GDRIVE_USER_CREDENTIALS_DATA, @@ -102,11 +101,14 @@ def test_09_SaveLoadCredentialsUsesDefaultStorage(mocker, tmp_path): auth.credentials = Credentials( "token", client_id="id", client_secret="secret" ) - spy = mocker.spy(FileStorage, "__init__") + storage = auth._default_storage + save = mocker.spy(storage, "put") + load = mocker.spy(storage, "get") auth.SaveCredentials() auth.LoadCredentials() assert auth.credentials.token == "token" - assert spy.call_count == 0 + save.assert_called_once() + load.assert_called_once_with() @pytest.mark.drive From 04ca8b6668f1050fb0a1ba44978cbece0d906ae8 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Fri, 20 Nov 2026 09:27:52 -0800 Subject: [PATCH 08/15] Share credential persistence across file and dictionary logins --- docs/oauth.rst | 4 +- pydrive2/_credentials.py | 123 ++++++++++++++++++++++++ pydrive2/auth.py | 156 ++++++------------------------- pydrive2/test/test_auth_flows.py | 104 ++++++++++++++++----- pydrive2/test/test_oauth.py | 6 +- 5 files changed, 241 insertions(+), 152 deletions(-) create mode 100644 pydrive2/_credentials.py diff --git a/docs/oauth.rst b/docs/oauth.rst index 160f8a4..33defad 100644 --- a/docs/oauth.rst +++ b/docs/oauth.rst @@ -142,7 +142,9 @@ Fields explained: Service account `.json` key file loaded into a string. **Required**: No. :save_credentials (bool): - Save OAuth user credentials after signing in. + Save OAuth user credentials after signing in. A fresh login stays in + memory when this setting is false. When you load saved credentials, + PyDrive2 writes refreshed tokens back to the same file or dictionary key. **Default**: False. **Required**: No. :save_credentials_backend (str): diff --git a/pydrive2/_credentials.py b/pydrive2/_credentials.py new file mode 100644 index 0000000..00a5b05 --- /dev/null +++ b/pydrive2/_credentials.py @@ -0,0 +1,123 @@ +import json +import os +import tempfile +from pathlib import Path + +from google.oauth2.credentials import Credentials + + +class InvalidCredentialsError(IOError): + """Error trying to read or write credentials.""" + + +class CredentialStore: + def __init__(self, destination: Path | tuple[dict[str, str], str], lock): + self.destination = destination + self.lock = lock + + @property + def backend(self): + return "file" if isinstance(self.destination, Path) else "dictionary" + + def load(self): + with self.lock: + if isinstance(self.destination, Path): + serialized = _read_file(self.destination) + else: + dictionary, key = self.destination + serialized = dictionary.get(key) + if not serialized: + return None + + try: + info = json.loads(serialized) + if ( + not isinstance(info, dict) + or info.get("type", "authorized_user") != "authorized_user" + or "_class" in info + or "access_token" in info + or not (info.get("token") or info.get("refresh_token")) + ): + raise ValueError("Expected google-auth user credentials") + # google-auth omits this key when saving online-only logins, + # but requires it to be present when loading them. + info.setdefault("refresh_token", None) + credentials = StoredUserCredentials.from_authorized_user_info( + info + ) + except (AttributeError, TypeError, ValueError) as exc: + raise InvalidCredentialsError( + "Select a fresh user credential file or dictionary key and " + "sign in again with PyDrive2 2.x." + ) from exc + credentials.store = self + return credentials + + def save(self, credentials: Credentials): + with self.lock: + serialized = credentials.to_json() + if isinstance(self.destination, Path): + _write_file(self.destination, serialized) + else: + dictionary, key = self.destination + dictionary[key] = serialized + + +class StoredUserCredentials(Credentials): + store: CredentialStore | None = None + + @classmethod + def from_flow(cls, credentials, store): + user = cls( + token=credentials.token, + refresh_token=credentials.refresh_token, + id_token=credentials.id_token, + token_uri=credentials.token_uri, + client_id=credentials.client_id, + client_secret=credentials.client_secret, + scopes=credentials.scopes, + expiry=credentials.expiry, + granted_scopes=credentials.granted_scopes, + ) + user.store = store + return user + + def refresh(self, request): + store = self.store + if store is None: + return super().refresh(request) + # AuthorizedHttp can refresh during a request, including after a 401. + previous_token = self.token + with store.lock: + if self.token == previous_token or not self.valid: + super().refresh(request) + store.save(self) + + +def _check_path(path): + if path.is_symlink(): + raise InvalidCredentialsError( + "Credentials file cannot be symbolic link" + ) + + +def _read_file(path): + _check_path(path) + try: + fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + except FileNotFoundError: + return None + with os.fdopen(fd, encoding="utf-8") as stream: + return stream.read() + + +def _write_file(path, serialized): + _check_path(path) + path.parent.mkdir(parents=True, exist_ok=True) + fd, name = tempfile.mkstemp(dir=path.parent) + try: + with os.fdopen(fd, "w", encoding="utf-8") as stream: + stream.write(serialized) + os.replace(name, path) + finally: + Path(name).unlink(missing_ok=True) diff --git a/pydrive2/auth.py b/pydrive2/auth.py index c75de33..5638ea9 100644 --- a/pydrive2/auth.py +++ b/pydrive2/auth.py @@ -1,7 +1,6 @@ import errno import json import os -import tempfile import threading import webbrowser from functools import wraps @@ -19,6 +18,11 @@ from googleapiclient.discovery import build from oauthlib.oauth2 import AccessDeniedError, OAuth2Error +from ._credentials import ( + CredentialStore, + InvalidCredentialsError, + StoredUserCredentials, +) from .apiattr import ApiAttribute, ApiAttributeMixin from .settings import ( InvalidConfigError, @@ -40,81 +44,10 @@ class AuthenticationError(AuthError): """General authentication error.""" -class InvalidCredentialsError(IOError): - """Error trying to read or write credentials.""" - - class RefreshError(AuthError): """Access token refresh error.""" -class _UserCredentials(user_credentials.Credentials): - storage = None - - def refresh(self, request): - storage = self.storage - if storage is None: - return super().refresh(request) - - # AuthorizedHttp can refresh during a request, including after a 401. - # Persist rotated tokens on that path as well as on explicit Refresh(). - previous_token = self.token - with storage.lock: - if self.token == previous_token or not self.valid: - super().refresh(request) - storage.put(self) - - -class _FileStorage: - def __init__(self, filename): - self.path = Path(filename) - self.lock = threading.RLock() - - def _check_path(self): - if self.path.is_symlink(): - raise InvalidCredentialsError( - "Credentials file cannot be symbolic link" - ) - - def get(self): - with self.lock: - self._check_path() - flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) - try: - fd = os.open(self.path, flags) - except FileNotFoundError: - return None - with os.fdopen(fd, encoding="utf-8") as stream: - return stream.read() - - def put(self, credentials): - with self.lock: - self._check_path() - self.path.parent.mkdir(parents=True, exist_ok=True) - fd, name = tempfile.mkstemp(dir=self.path.parent) - try: - with os.fdopen(fd, "w", encoding="utf-8") as stream: - stream.write(credentials.to_json()) - os.replace(name, self.path) - finally: - Path(name).unlink(missing_ok=True) - - -class _DictionaryStorage: - def __init__(self, dictionary, key): - self.dictionary = dictionary - self.key = key - self.lock = threading.RLock() - - def get(self): - with self.lock: - return self.dictionary.get(self.key) - - def put(self, credentials): - with self.lock: - self.dictionary[self.key] = credentials.to_json() - - def LoadAuth(decoratee): """Decorator to check if the auth is valid and loads auth if not.""" @@ -220,8 +153,12 @@ def __init__( self.settings = settings or self.DEFAULT_SETTINGS ValidateSettings(self.settings) - self._configured_storage = self._storage_from_settings() - self._default_storage = self._configured_storage + self._configured_store = self._storage_from_settings() + self._login_store = ( + self._configured_store + if self.settings.get("save_credentials") + else None + ) @property def access_token_expired(self): @@ -232,18 +169,12 @@ def access_token_expired(self): return self.credentials is None or not self.credentials.valid def _use_flow_credentials(self, credentials): - self.credentials = _UserCredentials( - token=credentials.token, - refresh_token=credentials.refresh_token, - id_token=credentials.id_token, - token_uri=credentials.token_uri, - client_id=credentials.client_id, - client_secret=credentials.client_secret, - scopes=credentials.scopes, - expiry=credentials.expiry, - granted_scopes=credentials.granted_scopes, - ) - self.credentials.storage = self._default_storage + if self._login_store is None: + self.credentials = credentials + else: + self.credentials = StoredUserCredentials.from_flow( + credentials, self._login_store + ) def LocalWebserverAuth( self, @@ -387,7 +318,7 @@ def _storage_from_settings(self): raise InvalidConfigError( "Please specify credentials file to read" ) - return _FileStorage(filename) + return CredentialStore(Path(filename), self._refresh_lock) if backend == "dictionary": dictionary = self.settings.get("save_credentials_dict") key = self.settings.get("save_credentials_key") @@ -395,7 +326,7 @@ def _storage_from_settings(self): raise InvalidConfigError("Please specify credentials dict") if key is None: raise InvalidConfigError("Please specify credentials key") - return _DictionaryStorage(dictionary, key) + return CredentialStore((dictionary, key), self._refresh_lock) if self.settings.get("save_credentials"): raise InvalidConfigError( f"Unknown save_credentials_backend: {backend}" @@ -413,18 +344,13 @@ def _credential_backend(self, backend): def _storage_for(self, backend, filename=None): if filename is not None: - return _FileStorage(filename) - if backend == "file": - if isinstance(self._configured_storage, _FileStorage): - return self._configured_storage + return CredentialStore(Path(filename), self._refresh_lock) + store = self._configured_store + if store is None or store.backend != backend: raise InvalidConfigError( - "Set save_credentials_file or pass an explicit credentials file" + f"Credential backend {backend!r} is not configured" ) - if isinstance(self._configured_storage, _DictionaryStorage): - return self._configured_storage - raise InvalidConfigError( - "Set save_credentials_dict and save_credentials_key" - ) + return store def LoadCredentials(self, backend=None): """Loads credentials or create empty credentials if it doesn't exist. @@ -449,32 +375,10 @@ def LoadCredentialsFile(self, credentials_file=None): """ self._load_credentials(self._storage_for("file", credentials_file)) - def _load_credentials(self, storage): - self._default_storage = storage - serialized = storage.get() - if not serialized: - self.credentials = None - return - try: - info = json.loads(serialized) - if ( - not isinstance(info, dict) - or info.get("type", "authorized_user") != "authorized_user" - or "_class" in info - or "access_token" in info - or not (info.get("token") or info.get("refresh_token")) - ): - raise ValueError("Expected google-auth user credentials") - # Online-only logins have an access token but no refresh token. - info.setdefault("refresh_token", None) - cred = _UserCredentials.from_authorized_user_info(info) - except (AttributeError, TypeError, ValueError) as exc: - raise InvalidCredentialsError( - "Select a fresh user credential file or dictionary key and sign " - "in again with PyDrive2 2.x." - ) from exc - cred.storage = storage - self.credentials = cred + def _load_credentials(self, store): + with self._refresh_lock: + self.credentials = store.load() + self._login_store = store def SaveCredentials(self, backend=None): """Saves credentials according to specified backend. @@ -505,7 +409,7 @@ def _save_credentials(self, backend, filename=None): raise InvalidCredentialsError( "Only signed-in user credentials can be saved" ) - self._storage_for(backend, filename).put(self.credentials) + self._storage_for(backend, filename).save(self.credentials) def LoadClientConfig(self, backend=None): """Loads client configuration according to specified backend. @@ -674,7 +578,7 @@ def Refresh(self): if ( user and self.settings.get("save_credentials") - and getattr(self.credentials, "storage", None) is None + and getattr(self.credentials, "store", None) is None ): self.SaveCredentials() diff --git a/pydrive2/test/test_auth_flows.py b/pydrive2/test/test_auth_flows.py index 94fa1f4..e36f224 100644 --- a/pydrive2/test/test_auth_flows.py +++ b/pydrive2/test/test_auth_flows.py @@ -2,6 +2,7 @@ import json import webbrowser from concurrent.futures import ThreadPoolExecutor +from datetime import datetime from threading import Barrier from urllib.parse import parse_qs, urlparse @@ -12,7 +13,7 @@ from cryptography.hazmat.primitives.serialization import pkcs12 from google.auth import jwt from google.oauth2.credentials import Credentials -from google_auth_httplib2 import AuthorizedHttp +from google_auth_httplib2 import AuthorizedHttp, Request from pydrive2.auth import ( AuthenticationError, @@ -68,30 +69,32 @@ def request(http, uri, method="GET", body=None, **kwargs): return calls -def test_automatic_http_refresh_is_saved(monkeypatch): - saved = { - "user": json.dumps( - { - "token": "old", - "expiry": "2099-01-01T00:00:00Z", - "refresh_token": "refresh", - "client_id": "id", - "client_secret": "secret", - "token_uri": TOKEN_URI, - } - ) +@pytest.mark.parametrize("backend", ["file", "dictionary"]) +def test_automatic_http_refresh_is_saved(backend, monkeypatch, tmp_path): + settings = { + "client_config_backend": "file", + "save_credentials": True, + "save_credentials_backend": backend, } - auth = GoogleAuth( - settings={ - "client_config_backend": "file", - "save_credentials": True, - "save_credentials_backend": "dictionary", - "save_credentials_dict": saved, - "save_credentials_key": "user", - } + if backend == "file": + settings["save_credentials_file"] = str(tmp_path / "user.json") + else: + settings.update(save_credentials_dict={}, save_credentials_key="user") + auth = GoogleAuth(settings=settings) + auth.credentials = Credentials( + "old", + refresh_token="refresh", + client_id="id", + client_secret="secret", + token_uri=TOKEN_URI, + expiry=datetime(2099, 1, 1), ) + auth.SaveCredentials() auth.LoadCredentials() + export = tmp_path / "export.json" + auth.SaveCredentialsFile(export) http = auth.Get_Http_Object() + access_tokens = [] def request(uri, method="GET", headers=None, **kwargs): if uri == TOKEN_URI: @@ -105,12 +108,53 @@ def request(uri, method="GET", headers=None, **kwargs): } ).encode(), ) + access_tokens.append(headers["authorization"]) status = "200" if headers["authorization"] == "Bearer new" else "401" return httplib2.Response({"status": status}), b"{}" monkeypatch.setattr(http.http, "request", request) assert http.request("https://example.test/drive")[0].status == 200 - assert json.loads(saved["user"])["refresh_token"] == "rotated" + assert access_tokens == ["Bearer old", "Bearer new"] + reopened = GoogleAuth(settings=settings) + reopened.LoadCredentials() + assert reopened.credentials.refresh_token == "rotated" + assert json.loads(export.read_text())["refresh_token"] == "refresh" + + +@pytest.mark.parametrize("save_credentials", [False, True]) +def test_login_respects_save_credentials( + save_credentials, tmp_path, mocker, token_endpoint +): + path = tmp_path / "user.json" + auth = GoogleAuth( + settings={ + "client_config_backend": "file", + "save_credentials": save_credentials, + "save_credentials_backend": "file", + "save_credentials_file": str(path), + } + ) + auth.flow = mocker.Mock() + auth.flow.run_local_server.return_value = Credentials( + "initial", + refresh_token="refresh", + client_id="id", + client_secret="secret", + token_uri=TOKEN_URI, + ) + auth.LocalWebserverAuth(launch_browser=False) + assert path.exists() == save_credentials + # The transport can refresh the login before a subsequent LoadCredentials. + http = auth.Get_Http_Object() + http.credentials.refresh(Request(http.http)) + assert len(token_endpoint) == 1 + if save_credentials: + assert json.loads(path.read_text())["token"] == "fresh" + else: + assert not path.exists() + + auth.SaveCredentials() + assert json.loads(path.read_text())["token"] == "fresh" def test_refresh_only_credentials_round_trip(tmp_path, token_endpoint): @@ -312,6 +356,22 @@ def test_custom_auth_and_online_only_cache(tmp_path, mocker): assert other.credentials.refresh_token is None +def test_failed_file_save_preserves_previous_credentials(tmp_path, mocker): + path = tmp_path / "user.json" + auth = GoogleAuth(settings_file=None) + auth.credentials = Credentials( + "original", client_id="id", client_secret="secret" + ) + auth.SaveCredentialsFile(path) + auth.credentials = Credentials("updated") + mocker.patch("os.replace", side_effect=OSError("replace failed")) + with pytest.raises(OSError, match="replace failed"): + auth.SaveCredentialsFile(path) + auth.LoadCredentialsFile(path) + assert auth.credentials.token == "original" + assert list(tmp_path.iterdir()) == [path] + + def test_credentials_file_rejects_symlinks(tmp_path): auth = GoogleAuth(settings_file=None) auth.credentials = Credentials("test") diff --git a/pydrive2/test/test_oauth.py b/pydrive2/test/test_oauth.py index 5187f5d..359adc7 100644 --- a/pydrive2/test/test_oauth.py +++ b/pydrive2/test/test_oauth.py @@ -101,9 +101,9 @@ def test_09_SaveLoadCredentialsUsesDefaultStorage(mocker, tmp_path): auth.credentials = Credentials( "token", client_id="id", client_secret="secret" ) - storage = auth._default_storage - save = mocker.spy(storage, "put") - load = mocker.spy(storage, "get") + storage = auth._configured_store + save = mocker.spy(storage, "save") + load = mocker.spy(storage, "load") auth.SaveCredentials() auth.LoadCredentials() assert auth.credentials.token == "token" From cea1540bb7ec096e9a288e5cb28f976758534b4b Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Fri, 20 Nov 2026 17:45:58 -0800 Subject: [PATCH 09/15] Use a consistent return in credential refresh --- pydrive2/_credentials.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pydrive2/_credentials.py b/pydrive2/_credentials.py index 00a5b05..f960906 100644 --- a/pydrive2/_credentials.py +++ b/pydrive2/_credentials.py @@ -85,7 +85,8 @@ def from_flow(cls, credentials, store): def refresh(self, request): store = self.store if store is None: - return super().refresh(request) + super().refresh(request) + return # AuthorizedHttp can refresh during a request, including after a 401. previous_token = self.token with store.lock: From d4fae0c3d9596d0b369be5101186f6e6bd8ae7b9 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Fri, 20 Nov 2026 17:58:38 -0800 Subject: [PATCH 10/15] Preserve credential permissions and coordinate OAuth cache refreshes --- docs/oauth.rst | 10 +- pydrive2/_credentials.py | 182 ++++++++++++---- pydrive2/_windows_credentials.py | 76 +++++++ pydrive2/auth.py | 49 +++-- pydrive2/test/test_auth_flows.py | 346 ++++++++++++++++++++++++++++++- pyproject.toml | 4 +- 6 files changed, 602 insertions(+), 65 deletions(-) create mode 100644 pydrive2/_windows_credentials.py diff --git a/docs/oauth.rst b/docs/oauth.rst index aac55b3..b476700 100644 --- a/docs/oauth.rst +++ b/docs/oauth.rst @@ -149,7 +149,9 @@ Fields explained: **Default**: 'file'. **Required**: No. :save_credentials_file (str): - Destination of credentials file. + Destination of the credentials file. Give the application a writable + directory for the lock and atomic file replacement. In containers, mount + the cache directory so the file can be replaced on refresh. **Required**: Yes, only if *save_credentials_backend* is 'file'. :save_credentials_dict (dict): Dict to use for storing credentials. @@ -214,7 +216,11 @@ customize an authentication flow as follows: 2. Direct the user to the authorization URL. Receive the authorization code at the redirect URI registered for your application and verify the OAuth state in the callback. -3. Pass the authorization code to :meth:`~pydrive2.auth.GoogleAuth.Auth`. +3. Pass the authorization code to :meth:`~pydrive2.auth.GoogleAuth.Auth` on + the same instance. Keep the flow for each login until the callback so the + exchange uses its original PKCE verifier. If a separate worker handles + the callback, retain the state and verifier in the user's server-side + session and restore the verifier on the flow before exchanging the code. Your *settings.yaml* will work for your customized authentication flow, too. diff --git a/pydrive2/_credentials.py b/pydrive2/_credentials.py index f960906..d94f567 100644 --- a/pydrive2/_credentials.py +++ b/pydrive2/_credentials.py @@ -1,61 +1,99 @@ import json import os +import sys import tempfile +from hashlib import sha256 from pathlib import Path +from threading import Lock, RLock +from uuid import uuid4 +from weakref import WeakValueDictionary +from filelock import FileLock from google.oauth2.credentials import Credentials +_dictionary_locks: WeakValueDictionary[tuple[int, str], RLock] = ( + WeakValueDictionary() +) +_registry_lock = Lock() + class InvalidCredentialsError(IOError): """Error trying to read or write credentials.""" class CredentialStore: - def __init__(self, destination: Path | tuple[dict[str, str], str], lock): + def __init__(self, destination: Path | tuple[dict[str, str], str]): self.destination = destination - self.lock = lock + self.lock: FileLock | RLock + if isinstance(destination, Path): + self.lock = FileLock(str(destination) + ".lock", mode=0o600) + else: + dictionary, key = destination + with _registry_lock: + self.lock = _dictionary_locks.setdefault( + (id(dictionary), key), RLock() + ) @property def backend(self): return "file" if isinstance(self.destination, Path) else "dictionary" - def load(self): - with self.lock: - if isinstance(self.destination, Path): - serialized = _read_file(self.destination) - else: - dictionary, key = self.destination - serialized = dictionary.get(key) - if not serialized: - return None - - try: - info = json.loads(serialized) - if ( - not isinstance(info, dict) - or info.get("type", "authorized_user") != "authorized_user" - or "_class" in info - or "access_token" in info - or not (info.get("token") or info.get("refresh_token")) - ): - raise ValueError("Expected google-auth user credentials") - # google-auth omits this key when saving online-only logins, - # but requires it to be present when loading them. + def load(self) -> "StoredUserCredentials | None": + if isinstance(self.destination, Path): + serialized = _read_file(self.destination) + else: + dictionary, key = self.destination + serialized = dictionary.get(key) + if not serialized: + return None + + try: + info = json.loads(serialized) + if ( + not isinstance(info, dict) + or info.get("type", "authorized_user") != "authorized_user" + or "_class" in info + or "access_token" in info + or not (info.get("token") or info.get("refresh_token")) + ): + raise ValueError("Expected google-auth user credentials") + if not info.get("refresh_token"): + # The SDK requires these keys even for access-token-only JSON. info.setdefault("refresh_token", None) - credentials = StoredUserCredentials.from_authorized_user_info( - info - ) - except (AttributeError, TypeError, ValueError) as exc: - raise InvalidCredentialsError( - "Select a fresh user credential file or dictionary key and " - "sign in again with PyDrive2 2.x." - ) from exc - credentials.store = self - return credentials + info.setdefault("client_id", None) + info.setdefault("client_secret", None) + credentials = StoredUserCredentials.from_authorized_user_info(info) + if not info.get("expiry") and not credentials.refresh_token: + credentials.expiry = None + credentials._session_id = info.get( + "_pydrive2_session" + ) or _token_identity(credentials) + except (AttributeError, TypeError, ValueError) as exc: + raise InvalidCredentialsError( + "Select a fresh user credential file or dictionary key and " + "sign in again with PyDrive2 2.x." + ) from exc + credentials.store = self + return credentials def save(self, credentials: Credentials): + if credentials.refresh_token: + if not credentials.client_id or not credentials.client_secret: + raise InvalidCredentialsError( + "Saving a refresh token requires a client ID and client secret" + ) + elif not credentials.token: + raise InvalidCredentialsError("No user token to save") + if isinstance(self.destination, Path): + self.destination.parent.mkdir(parents=True, exist_ok=True) with self.lock: serialized = credentials.to_json() + session = getattr(credentials, "_session_id", None) + if session and session != _token_identity(credentials): + # Keep the grant identifiable when its refresh token rotates. + info = json.loads(serialized) + info["_pydrive2_session"] = session + serialized = json.dumps(info) if isinstance(self.destination, Path): _write_file(self.destination, serialized) else: @@ -63,8 +101,16 @@ def save(self, credentials: Credentials): dictionary[key] = serialized +def _token_identity(credentials): + return sha256( + (credentials.refresh_token or credentials.token).encode() + ).hexdigest() + + class StoredUserCredentials(Credentials): store: CredentialStore | None = None + _session_id: str | None = None + _pending_refresh: "StoredUserCredentials | None" = None @classmethod def from_flow(cls, credentials, store): @@ -80,8 +126,22 @@ def from_flow(cls, credentials, store): granted_scopes=credentials.granted_scopes, ) user.store = store + user._session_id = _token_identity(user) return user + def __getstate__(self): + state = super().__getstate__() + state.pop("store", None) + state.pop("_pending_refresh", None) + return state + + def use_client_config(self, config): + # google-auth discards token_uri from cache JSON. Only a matching + # application configuration may override its trusted Google endpoint. + endpoint = config.get("token_uri") + if endpoint and config.get("client_id") == self.client_id: + self._token_uri = endpoint + def refresh(self, request): store = self.store if store is None: @@ -90,9 +150,43 @@ def refresh(self, request): # AuthorizedHttp can refresh during a request, including after a 401. previous_token = self.token with store.lock: - if self.token == previous_token or not self.valid: + if self.token != previous_token and self.valid: + return + cached = store.load() + if ( + cached is None + or cached._session_id != self._session_id + or cached.client_id != self.client_id + ): + # A different login or an explicit deletion owns this location. + self.store = None super().refresh(request) - store.save(self) + return + # If publication failed after rotation, the old refresh token may + # already be invalid. Retry the write before another exchange. + pending = self._pending_refresh + if pending is not None and cached.token == self.token: + cached = pending + else: + pending = None + if not cached.valid or ( + pending is None and cached.token == previous_token + ): + # Only the original client's trusted endpoint may receive tokens. + cached._token_uri = self.token_uri + Credentials.refresh(cached, request) + pending = self._pending_refresh = cached + if pending is not None: + store.save(pending) + self._pending_refresh = None + self.token = cached.token + self.expiry = cached.expiry + self._refresh_token = cached.refresh_token + self._rapt_token = cached.rapt_token + if cached.id_token is not None: + self._id_token = cached.id_token + if cached.granted_scopes is not None: + self._granted_scopes = cached.granted_scopes def _check_path(path): @@ -105,7 +199,12 @@ def _check_path(path): def _read_file(path): _check_path(path) try: - fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + if sys.platform == "win32": + from ._windows_credentials import open_for_read + + fd = open_for_read(path) + else: + fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) except FileNotFoundError: return None with os.fdopen(fd, encoding="utf-8") as stream: @@ -114,8 +213,13 @@ def _read_file(path): def _write_file(path, serialized): _check_path(path) - path.parent.mkdir(parents=True, exist_ok=True) - fd, name = tempfile.mkstemp(dir=path.parent) + if sys.platform == "win32": + from ._windows_credentials import create_for_replace + + name = path.with_name(uuid4().hex + ".tmp") + fd = create_for_replace(path, name) + else: + fd, name = tempfile.mkstemp(dir=path.parent) try: with os.fdopen(fd, "w", encoding="utf-8") as stream: stream.write(serialized) diff --git a/pydrive2/_windows_credentials.py b/pydrive2/_windows_credentials.py new file mode 100644 index 0000000..7551af1 --- /dev/null +++ b/pydrive2/_windows_credentials.py @@ -0,0 +1,76 @@ +import os +import sys + +assert sys.platform == "win32" + +import msvcrt + +import pywintypes +import win32con +import win32file +import win32security +import winerror + +from ._credentials import InvalidCredentialsError + + +def _fileno(handle, flags): + fd = msvcrt.open_osfhandle(int(handle), flags) + handle.Detach() # The CRT descriptor now owns the Windows handle. + return fd + + +def open_for_read(path): + try: + handle = win32file.CreateFile( + str(path), + win32con.GENERIC_READ, + win32con.FILE_SHARE_READ | win32con.FILE_SHARE_DELETE, + None, + win32con.OPEN_EXISTING, + win32file.FILE_FLAG_OPEN_REPARSE_POINT, + None, + ) + except pywintypes.error as exc: + if exc.winerror in ( + winerror.ERROR_FILE_NOT_FOUND, + winerror.ERROR_PATH_NOT_FOUND, + ): + raise FileNotFoundError(path) from exc + raise + attributes = win32file.GetFileInformationByHandle(handle)[0] + if attributes & win32con.FILE_ATTRIBUTE_REPARSE_POINT: + handle.Close() + raise InvalidCredentialsError( + "Credentials file cannot be symbolic link" + ) + return _fileno(handle, os.O_RDONLY) + + +def create_for_replace(path, temporary): + # Apply the DACL at creation, before another process can open the file. + if path.exists(): + descriptor = win32security.GetFileSecurity( + str(path), win32security.DACL_SECURITY_INFORMATION + ) + protected = win32security.SE_DACL_PROTECTED + descriptor.SetSecurityDescriptorControl(protected, protected) + else: + descriptor = ( + win32security.ConvertStringSecurityDescriptorToSecurityDescriptor( + "D:P(A;;FA;;;OW)", win32security.SDDL_REVISION_1 + ) + ) + security = pywintypes.SECURITY_ATTRIBUTES() + security.SECURITY_DESCRIPTOR = descriptor + security.bInheritHandle = False + handle = win32file.CreateFile( + str(temporary), + win32con.GENERIC_WRITE, + 0, + security, + win32con.CREATE_NEW, + win32con.FILE_ATTRIBUTE_NORMAL, + None, + ) + return _fileno(handle, os.O_WRONLY) diff --git a/pydrive2/auth.py b/pydrive2/auth.py index 5638ea9..d1a58c7 100644 --- a/pydrive2/auth.py +++ b/pydrive2/auth.py @@ -154,11 +154,7 @@ def __init__( ValidateSettings(self.settings) self._configured_store = self._storage_from_settings() - self._login_store = ( - self._configured_store - if self.settings.get("save_credentials") - else None - ) + self._login_store = self._configured_store @property def access_token_expired(self): @@ -169,12 +165,12 @@ def access_token_expired(self): return self.credentials is None or not self.credentials.valid def _use_flow_credentials(self, credentials): - if self._login_store is None: - self.credentials = credentials - else: - self.credentials = StoredUserCredentials.from_flow( + if self.settings.get("save_credentials"): + credentials = StoredUserCredentials.from_flow( credentials, self._login_store ) + self._login_store.save(credentials) + self.credentials = credentials def LocalWebserverAuth( self, @@ -205,12 +201,13 @@ def LocalWebserverAuth( :raises: AuthenticationRejected, AuthenticationError """ if self.credentials is None and self.settings.get("save_credentials"): - self.LoadCredentials() + self._load_credentials(self._login_store) if not self.access_token_expired: return if self.credentials is not None and ( not isinstance(self.credentials, user_credentials.Credentials) or self.credentials.refresh_token + or self.credentials.refresh_handler ): self._refresh_if_expired() return @@ -250,8 +247,6 @@ def LocalWebserverAuth( ) from exc self._use_flow_credentials(credentials) self.Authorize() - if self.settings.get("save_credentials"): - self.SaveCredentials() return raise AuthenticationError( "Could not bind to any configured OAuth callback port" @@ -318,7 +313,7 @@ def _storage_from_settings(self): raise InvalidConfigError( "Please specify credentials file to read" ) - return CredentialStore(Path(filename), self._refresh_lock) + return CredentialStore(Path(filename)) if backend == "dictionary": dictionary = self.settings.get("save_credentials_dict") key = self.settings.get("save_credentials_key") @@ -326,7 +321,7 @@ def _storage_from_settings(self): raise InvalidConfigError("Please specify credentials dict") if key is None: raise InvalidConfigError("Please specify credentials key") - return CredentialStore((dictionary, key), self._refresh_lock) + return CredentialStore((dictionary, key)) if self.settings.get("save_credentials"): raise InvalidConfigError( f"Unknown save_credentials_backend: {backend}" @@ -344,7 +339,7 @@ def _credential_backend(self, backend): def _storage_for(self, backend, filename=None): if filename is not None: - return CredentialStore(Path(filename), self._refresh_lock) + return CredentialStore(Path(filename)) store = self._configured_store if store is None or store.backend != backend: raise InvalidConfigError( @@ -377,7 +372,20 @@ def LoadCredentialsFile(self, credentials_file=None): def _load_credentials(self, store): with self._refresh_lock: - self.credentials = store.load() + credentials = store.load() + if credentials and credentials.refresh_token: + config = self.client_config or self.settings.get( + "client_config", {} + ) + if ( + not config + and self.settings["client_config_backend"] == "file" + and Path(self.settings["client_config_file"]).is_file() + ): + self.LoadClientConfigFile() + config = self.client_config + credentials.use_client_config(config) + self.credentials = credentials self._login_store = store def SaveCredentials(self, backend=None): @@ -557,7 +565,10 @@ def Refresh(self): if self.credentials is None: raise RefreshError("No credential to refresh.") user = isinstance(self.credentials, user_credentials.Credentials) - if user and self.credentials.refresh_token is None: + if user and not ( + self.credentials.refresh_token + or self.credentials.refresh_handler + ): raise RefreshError( "No refresh_token found. " "Please set access_type of OAuth to offline." @@ -578,9 +589,9 @@ def Refresh(self): if ( user and self.settings.get("save_credentials") - and getattr(self.credentials, "store", None) is None + and not isinstance(self.credentials, StoredUserCredentials) ): - self.SaveCredentials() + self._login_store.save(self.credentials) def GetAuthUrl(self): """Creates authentication url where user visits to grant access. diff --git a/pydrive2/test/test_auth_flows.py b/pydrive2/test/test_auth_flows.py index e36f224..84403f1 100644 --- a/pydrive2/test/test_auth_flows.py +++ b/pydrive2/test/test_auth_flows.py @@ -1,13 +1,20 @@ import errno import json +import os +import pickle +import sys import webbrowser +from base64 import urlsafe_b64encode from concurrent.futures import ThreadPoolExecutor from datetime import datetime +from hashlib import sha256 +from http.client import HTTPConnection from threading import Barrier -from urllib.parse import parse_qs, urlparse +from urllib.parse import parse_qs, urlencode, urlparse import httplib2 import pytest +import requests from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives.serialization import pkcs12 @@ -20,6 +27,7 @@ GoogleAuth, InvalidCredentialsError, ) +from pydrive2.fs.spec import GDriveAuthError, _client_auth TOKEN_URI = "https://oauth2.googleapis.com/token" EMAIL = "service@example.test" @@ -134,6 +142,7 @@ def test_login_respects_save_credentials( "save_credentials_file": str(path), } ) + auth.LoadCredentialsFile(path) auth.flow = mocker.Mock() auth.flow.run_local_server.return_value = Credentials( "initial", @@ -177,6 +186,27 @@ def test_refresh_only_credentials_round_trip(tmp_path, token_endpoint): assert len(token_endpoint) == 1 +def test_refresh_retries_failed_cache_write(tmp_path, mocker, token_endpoint): + path = tmp_path / "user.json" + auth = GoogleAuth(settings_file=None) + auth.credentials = Credentials( + None, + refresh_token="refresh", + client_id="id", + client_secret="secret", + token_uri=TOKEN_URI, + ) + auth.SaveCredentialsFile(path) + auth.LoadCredentialsFile(path) + replace = mocker.patch("os.replace", side_effect=OSError("replace failed")) + with pytest.raises(OSError, match="replace failed"): + auth.Refresh() + mocker.stop(replace) + auth.Refresh() + assert len(token_endpoint) == 1 + assert json.loads(path.read_text())["token"] == "fresh" + + def test_concurrent_login_refreshes_and_saves_once(mocker, token_endpoint): saved = {} auth = GoogleAuth( @@ -328,9 +358,13 @@ def test_login_options_and_port_fallback(mocker): def test_custom_auth_and_online_only_cache(tmp_path, mocker): + path = tmp_path / "online" auth = GoogleAuth( settings={ "client_config_backend": "settings", + "save_credentials": True, + "save_credentials_backend": "file", + "save_credentials_file": str(path), "client_config": { "client_id": "id", "client_secret": "secret", @@ -348,14 +382,73 @@ def test_custom_auth_and_online_only_cache(tmp_path, mocker): } auth.Auth("code") auth.flow.fetch_token.assert_called_once_with(code="code") - path = tmp_path / "online" - auth.SaveCredentialsFile(path) other = GoogleAuth(settings_file=None) other.LoadCredentialsFile(path) assert other.credentials.valid assert other.credentials.refresh_token is None +def test_loopback_code_exchange(tmp_path, monkeypatch, mocker): + auth = GoogleAuth( + settings={ + "client_config_backend": "settings", + "client_config": { + "client_id": "client", + "client_secret": "secret", + }, + "get_refresh_token": True, + "save_credentials": True, + "save_credentials_backend": "file", + "save_credentials_file": str(tmp_path / "user.json"), + } + ) + challenge = None + callback = None + + def open_browser(url, **kwargs): + nonlocal challenge, callback + query = parse_qs(urlparse(url).query) + assert query["code_challenge_method"] == ["S256"] + challenge = query["code_challenge"][0] + redirect = urlparse(query["redirect_uri"][0]) + callback = HTTPConnection(redirect.hostname, redirect.port, timeout=5) + params = urlencode({"state": query["state"][0], "code": "approved"}) + callback.request("GET", f"{redirect.path}?{params}") + return True + + def token_response(session, request, **kwargs): + form = parse_qs(request.body) + assert form["code"] == ["approved"] + digest = sha256(form["code_verifier"][0].encode()).digest() + assert urlsafe_b64encode(digest).rstrip(b"=").decode() == challenge + response = requests.Response() + response.request = request + response.status_code = 200 + response._content = json.dumps( + { + "access_token": "access", + "refresh_token": "refresh", + "expires_in": 3600, + "token_type": "Bearer", + } + ).encode() + return response + + monkeypatch.setattr( + webbrowser, "get", lambda *args: mocker.Mock(open=open_browser) + ) + monkeypatch.setattr(requests.Session, "send", token_response) + try: + auth.LocalWebserverAuth(host_name="127.0.0.1", port_numbers=[0]) + assert callback.getresponse().status == 200 + finally: + if callback is not None: + callback.close() + auth.LoadCredentials() + assert auth.credentials.valid + assert auth.credentials.refresh_token == "refresh" + + def test_failed_file_save_preserves_previous_credentials(tmp_path, mocker): path = tmp_path / "user.json" auth = GoogleAuth(settings_file=None) @@ -363,13 +456,14 @@ def test_failed_file_save_preserves_previous_credentials(tmp_path, mocker): "original", client_id="id", client_secret="secret" ) auth.SaveCredentialsFile(path) + original_files = set(tmp_path.iterdir()) auth.credentials = Credentials("updated") mocker.patch("os.replace", side_effect=OSError("replace failed")) with pytest.raises(OSError, match="replace failed"): auth.SaveCredentialsFile(path) auth.LoadCredentialsFile(path) assert auth.credentials.token == "original" - assert list(tmp_path.iterdir()) == [path] + assert set(tmp_path.iterdir()) == original_files def test_credentials_file_rejects_symlinks(tmp_path): @@ -387,3 +481,247 @@ def test_credentials_file_rejects_symlinks(tmp_path): with pytest.raises(InvalidCredentialsError): auth.LoadCredentialsFile(link) assert target.read_text() == "original" + + +def test_reauth_uses_explicit_profile(tmp_path, mocker): + default = tmp_path / "default.json" + profile = tmp_path / "profile.json" + auth = GoogleAuth( + settings={ + "client_config_backend": "file", + "save_credentials": True, + "save_credentials_backend": "file", + "save_credentials_file": str(default), + } + ) + auth.credentials = Credentials("default-user") + auth.SaveCredentials() + auth.credentials = Credentials("expired", expiry=datetime(2001, 1, 1)) + auth.SaveCredentialsFile(profile) + auth.LoadCredentialsFile(profile) + auth.flow = mocker.Mock() + auth.flow.run_local_server.return_value = Credentials("new-profile-user") + auth.LocalWebserverAuth() + assert json.loads(profile.read_text())["token"] == "new-profile-user" + assert json.loads(default.read_text())["token"] == "default-user" + + +@pytest.mark.parametrize("backend", ["file", "dictionary"]) +def test_shared_cache_refresh_rotation(backend, tmp_path, monkeypatch): + settings = { + "client_config_backend": "file", + "save_credentials": True, + "save_credentials_backend": backend, + } + if backend == "file": + settings["save_credentials_file"] = str(tmp_path / "user.json") + else: + settings.update(save_credentials_dict={}, save_credentials_key="user") + first, second = (GoogleAuth(settings=settings) for _ in range(2)) + first.credentials = Credentials( + None, + refresh_token="original", + client_id="id", + client_secret="secret", + token_uri=TOKEN_URI, + ) + first.SaveCredentials() + first.LoadCredentials() + second.LoadCredentials() + arrivals = Barrier(2) + token_requests = [] + + def request(http, uri, method, body=None, **kwargs): + assert uri == TOKEN_URI + assert parse_qs(body.decode())["refresh_token"] == ["original"] + token_requests.append(uri) + return httplib2.Response({"status": "200"}), json.dumps( + { + "access_token": "fresh", + "refresh_token": "rotated", + "expires_in": 3600, + } + ).encode() + + def refresh(auth): + arrivals.wait(timeout=10) + auth.Refresh() + + monkeypatch.setattr(httplib2.Http, "request", request) + with ThreadPoolExecutor(max_workers=2) as pool: + list(pool.map(refresh, [first, second])) + assert len(token_requests) == 1 + for auth in (first, second): + assert (auth.credentials.token, auth.credentials.refresh_token) == ( + "fresh", + "rotated", + ) + assert auth.credentials.valid + detached = pickle.loads(pickle.dumps(second.credentials)) + assert detached.refresh_token == "rotated" + assert detached.store is None + + +def test_refresh_does_not_overwrite_a_different_login( + tmp_path, token_endpoint +): + path = tmp_path / "user.json" + auth = GoogleAuth(settings_file=None) + auth.credentials = Credentials( + None, + refresh_token="first", + client_id="id", + client_secret="secret", + token_uri=TOKEN_URI, + ) + auth.SaveCredentialsFile(path) + auth.LoadCredentialsFile(path) + replacement = GoogleAuth(settings_file=None) + replacement.credentials = Credentials("second-user") + replacement.SaveCredentialsFile(path) + + auth.Refresh() + assert len(token_endpoint) == 1 + assert auth.credentials.token == "fresh" + replacement.LoadCredentialsFile(path) + assert replacement.credentials.token == "second-user" + + +@pytest.mark.parametrize("backend", ["settings", "file"]) +def test_token_endpoint_comes_from_client_configuration( + backend, tmp_path, monkeypatch +): + path = tmp_path / "user.json" + endpoint = "https://proxy.example.test/oauth/token" + config = { + "client_id": "id", + "client_secret": "secret", + "token_uri": endpoint, + } + settings = {"client_config_backend": backend} + if backend == "settings": + settings["client_config"] = config + else: + client_file = tmp_path / "client_secrets.json" + client_file.write_text( + json.dumps( + { + "web": { + **config, + "auth_uri": "https://accounts.example.test/auth", + "redirect_uris": ["http://localhost"], + } + } + ) + ) + settings["client_config_file"] = str(client_file) + auth = GoogleAuth(settings=settings) + auth.credentials = Credentials( + None, + refresh_token="refresh", + token_uri="https://cache-only.example.test/token", + client_id="id", + client_secret="secret", + ) + auth.SaveCredentialsFile(path) + other = GoogleAuth(settings=settings) + other.LoadCredentialsFile(path) + calls = [] + + def request(http, uri, **kwargs): + calls.append(uri) + return httplib2.Response( + {"status": "200"} + ), b'{"access_token":"fresh","expires_in":3600}' + + monkeypatch.setattr(httplib2.Http, "request", request) + other.Refresh() + assert calls == [endpoint] + + +@pytest.mark.parametrize("expiry", [None, datetime(2099, 1, 1)]) +def test_token_only_cache(tmp_path, expiry): + auth = GoogleAuth(settings_file=None) + auth.credentials = Credentials("token", expiry=expiry) + path = tmp_path / "user.json" + auth.SaveCredentialsFile(path) + auth.LoadCredentialsFile(path) + assert auth.credentials.valid + assert auth.credentials.expiry == expiry + + +def test_noninteractive_refresh_handler(monkeypatch): + monkeypatch.setenv("GDRIVE_NON_INTERACTIVE", "true") + auth = GoogleAuth(settings_file=None) + auth.credentials = Credentials( + None, + refresh_handler=lambda request, scopes=None: ( + "brokered", + datetime(2099, 1, 1), + ), + ) + auth.LocalWebserverAuth() + assert auth.credentials.token == "brokered" + assert auth.credentials.valid + + +def test_fsspec_upgrade_uses_new_cache(tmp_path, mocker): + mocker.patch("appdirs.user_cache_dir", return_value=str(tmp_path)) + legacy = json.dumps({"_class": "OAuth2Credentials", "access_token": "old"}) + old_path = tmp_path / "client" / "default.json" + old_path.parent.mkdir() + old_path.write_text(legacy) + login = mocker.patch( + "google_auth_oauthlib.flow.InstalledAppFlow.run_local_server", + return_value=Credentials("new-token"), + ) + for _ in range(2): + auth = _client_auth("client", "secret") + assert auth.credentials.token == "new-token" + login.assert_called_once() + assert (tmp_path / "v2" / "client" / "default.json").is_file() + for source in ( + {"client_json": legacy}, + {"client_json_file_path": str(old_path)}, + ): + with pytest.raises(GDriveAuthError, match="sign in again") as exc: + _client_auth("client", "secret", **source) + assert isinstance(exc.value.__cause__, InvalidCredentialsError) + assert old_path.read_text() == legacy + + +@pytest.mark.skipif(sys.platform != "win32", reason="Windows DACL") +def test_windows_cache_preserves_file_permissions(tmp_path, monkeypatch): + import win32security as security + + path = tmp_path / "user.json" + auth = GoogleAuth(settings_file=None) + auth.credentials = Credentials("first") + auth.SaveCredentialsFile(path) + flags = security.DACL_SECURITY_INFORMATION + + def dacl(filename): + sd = security.GetFileSecurity(str(filename), flags) + return security.ConvertSecurityDescriptorToStringSecurityDescriptor( + sd, 1, flags + ) + + assert "(A;;FA;;;OW)" in dacl(path) + # Add a distinct explicit grant so an always-owner-only replacement also fails. + sd = security.ConvertStringSecurityDescriptorToSecurityDescriptor( + "D:P(A;;FA;;;OW)(A;;FR;;;SY)", 1 + ) + security.SetFileSecurity(str(path), flags, sd) + original = dacl(path) + replace = os.replace + + def check_staged_acl(src, dst): + if dst == path: + assert dacl(src) == original + return replace(src, dst) + + monkeypatch.setattr(os, "replace", check_staged_acl) + auth.credentials = Credentials("updated") + auth.SaveCredentialsFile(path) + assert dacl(path) == original + assert json.loads(path.read_text())["token"] == "updated" diff --git a/pyproject.toml b/pyproject.toml index aa4a991..71886cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,11 +26,13 @@ dependencies = [ "google-api-python-client>=1.12.5", "google-auth>=2.15.0", "google-auth-httplib2>=0.2.0", - "google-auth-oauthlib>=1.2.0", + "google-auth-oauthlib>=1.5.0", + "filelock>=3.30.0", "httplib2>=0.19.0", "oauthlib>=3.0.0", "PyYAML>=3.0", "cryptography>=3.4", + "pywin32>=305; sys_platform == 'win32'", ] license = {text = "Apache License 2.0"} dynamic = ["version"] From 8fd207420faf1d3385e36294e5064c144ad96b38 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Fri, 20 Nov 2026 19:09:49 -0800 Subject: [PATCH 11/15] Skip the Windows Python 3.15 frozen-app test pending #399 --- pydrive2/__pyinstaller/test_hook-googleapiclient.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pydrive2/__pyinstaller/test_hook-googleapiclient.py b/pydrive2/__pyinstaller/test_hook-googleapiclient.py index 71f16fa..4fca8fa 100644 --- a/pydrive2/__pyinstaller/test_hook-googleapiclient.py +++ b/pydrive2/__pyinstaller/test_hook-googleapiclient.py @@ -1,5 +1,7 @@ import subprocess +import sys +import pytest from PyInstaller import __main__ as pyi_main _APP_SOURCE = """ @@ -13,6 +15,11 @@ """ +# TODO: Remove this skip once the PyInstaller fix is verified in #399. +@pytest.mark.skipif( + sys.platform == "win32" and sys.version_info[:2] == (3, 15), + reason="PyInstaller omits python3t.dll on Windows with Python 3.15", +) def test_pyi_hook_google_api_client(tmp_path): app_name = "userapp" workpath = tmp_path / "build" From baf7e6a67d24b5bf46eec40f6dad16ae0f838afe Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sat, 21 Nov 2026 09:59:31 -0800 Subject: [PATCH 12/15] Fix credential cache reads and isolate optional OAuth configuration --- pydrive2/_credentials.py | 52 ++++++++++---- pydrive2/_windows_credentials.py | 40 +---------- pydrive2/auth.py | 32 +++++---- pydrive2/test/test_auth_flows.py | 114 ++++++++++++++++++++++++++++++- 4 files changed, 171 insertions(+), 67 deletions(-) diff --git a/pydrive2/_credentials.py b/pydrive2/_credentials.py index d94f567..7c5f4d1 100644 --- a/pydrive2/_credentials.py +++ b/pydrive2/_credentials.py @@ -1,7 +1,10 @@ +import errno import json import os +import stat import sys import tempfile +from contextlib import ExitStack from hashlib import sha256 from pathlib import Path from threading import Lock, RLock @@ -40,10 +43,11 @@ def backend(self): def load(self) -> "StoredUserCredentials | None": if isinstance(self.destination, Path): - serialized = _read_file(self.destination) + serialized = _read_file(self.destination, self.lock) else: dictionary, key = self.destination - serialized = dictionary.get(key) + with self.lock: + serialized = dictionary.get(key) if not serialized: return None @@ -190,24 +194,44 @@ def refresh(self, request): def _check_path(path): - if path.is_symlink(): + try: + info = path.lstat() + except FileNotFoundError: + return None + # Windows uses the name-surrogate bit for symlinks and junctions. + if ( + stat.S_ISLNK(info.st_mode) + or getattr(info, "st_reparse_tag", 0) & 0x20000000 + ): raise InvalidCredentialsError( "Credentials file cannot be symbolic link" ) + return info -def _read_file(path): - _check_path(path) - try: - if sys.platform == "win32": - from ._windows_credentials import open_for_read - - fd = open_for_read(path) - else: - fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) - except FileNotFoundError: +def _read_file(path, lock): + if _check_path(path) is None: return None - with os.fdopen(fd, encoding="utf-8") as stream: + with ExitStack() as stack: + try: + stack.enter_context(lock) + except OSError as exc: + # A pre-provisioned cache can be mounted read-only. + if exc.errno not in (errno.EACCES, errno.EPERM, errno.EROFS): + raise + try: + fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + except FileNotFoundError: + return None + stream = stack.enter_context(os.fdopen(fd, encoding="utf-8")) + if sys.platform == "win32": + # Open normally so Windows can hydrate cloud or compressed files. + # Verify the opened file before reading since Windows lacks O_NOFOLLOW. + info = _check_path(path) + if info is None or not os.path.samestat(info, os.fstat(fd)): + raise InvalidCredentialsError( + "Credentials file changed while opening" + ) return stream.read() diff --git a/pydrive2/_windows_credentials.py b/pydrive2/_windows_credentials.py index 7551af1..c2edbe4 100644 --- a/pydrive2/_windows_credentials.py +++ b/pydrive2/_windows_credentials.py @@ -9,42 +9,6 @@ import win32con import win32file import win32security -import winerror - -from ._credentials import InvalidCredentialsError - - -def _fileno(handle, flags): - fd = msvcrt.open_osfhandle(int(handle), flags) - handle.Detach() # The CRT descriptor now owns the Windows handle. - return fd - - -def open_for_read(path): - try: - handle = win32file.CreateFile( - str(path), - win32con.GENERIC_READ, - win32con.FILE_SHARE_READ | win32con.FILE_SHARE_DELETE, - None, - win32con.OPEN_EXISTING, - win32file.FILE_FLAG_OPEN_REPARSE_POINT, - None, - ) - except pywintypes.error as exc: - if exc.winerror in ( - winerror.ERROR_FILE_NOT_FOUND, - winerror.ERROR_PATH_NOT_FOUND, - ): - raise FileNotFoundError(path) from exc - raise - attributes = win32file.GetFileInformationByHandle(handle)[0] - if attributes & win32con.FILE_ATTRIBUTE_REPARSE_POINT: - handle.Close() - raise InvalidCredentialsError( - "Credentials file cannot be symbolic link" - ) - return _fileno(handle, os.O_RDONLY) def create_for_replace(path, temporary): @@ -73,4 +37,6 @@ def create_for_replace(path, temporary): win32con.FILE_ATTRIBUTE_NORMAL, None, ) - return _fileno(handle, os.O_WRONLY) + fd = msvcrt.open_osfhandle(int(handle), os.O_WRONLY) + handle.Detach() # The CRT descriptor now owns the Windows handle. + return fd diff --git a/pydrive2/auth.py b/pydrive2/auth.py index d1a58c7..1af395d 100644 --- a/pydrive2/auth.py +++ b/pydrive2/auth.py @@ -3,6 +3,7 @@ import os import threading import webbrowser +from contextlib import suppress from functools import wraps from pathlib import Path from typing import ClassVar @@ -380,10 +381,12 @@ def _load_credentials(self, store): if ( not config and self.settings["client_config_backend"] == "file" - and Path(self.settings["client_config_file"]).is_file() ): - self.LoadClientConfigFile() - config = self.client_config + # A cached login can refresh without a client secrets file. + path = Path(self.settings["client_config_file"]) + with suppress(OSError): + if path.is_file(): + config = self._read_client_config_file(path) credentials.use_client_config(config) self.credentials = credentials self._login_store = store @@ -455,6 +458,10 @@ def LoadClientConfigFile(self, client_config_file=None): """ if client_config_file is None: client_config_file = self.settings["client_config_file"] + self.client_config = self._read_client_config_file(client_config_file) + + @staticmethod + def _read_client_config_file(client_config_file): try: with open(client_config_file, encoding="utf-8") as stream: config = json.load(stream) @@ -462,19 +469,14 @@ def LoadClientConfigFile(self, client_config_file=None): if client_type not in ("web", "installed"): raise ValueError("Expected web or installed client") info = config[client_type] - self.client_config = { - name: info[name] - for name in ( - "client_id", - "client_secret", - "auth_uri", - "token_uri", - ) + return { + "client_id": info["client_id"], + "client_secret": info["client_secret"], + "auth_uri": info["auth_uri"], + "token_uri": info["token_uri"], + "redirect_uri": info["redirect_uris"][0], + "revoke_uri": info.get("revoke_uri"), } - self.client_config.update( - redirect_uri=info["redirect_uris"][0], - revoke_uri=info.get("revoke_uri"), - ) except (OSError, KeyError, IndexError, TypeError, ValueError) as exc: raise InvalidConfigError("Invalid client secrets file") from exc diff --git a/pydrive2/test/test_auth_flows.py b/pydrive2/test/test_auth_flows.py index 84403f1..3755d4b 100644 --- a/pydrive2/test/test_auth_flows.py +++ b/pydrive2/test/test_auth_flows.py @@ -2,6 +2,8 @@ import json import os import pickle +import stat +import subprocess import sys import webbrowser from base64 import urlsafe_b64encode @@ -9,7 +11,7 @@ from datetime import datetime from hashlib import sha256 from http.client import HTTPConnection -from threading import Barrier +from threading import Barrier, Event from urllib.parse import parse_qs, urlencode, urlparse import httplib2 @@ -18,16 +20,19 @@ from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives.serialization import pkcs12 +from filelock import Timeout from google.auth import jwt from google.oauth2.credentials import Credentials from google_auth_httplib2 import AuthorizedHttp, Request +from pydrive2._credentials import CredentialStore from pydrive2.auth import ( AuthenticationError, GoogleAuth, InvalidCredentialsError, ) from pydrive2.fs.spec import GDriveAuthError, _client_auth +from pydrive2.settings import InvalidConfigError TOKEN_URI = "https://oauth2.googleapis.com/token" EMAIL = "service@example.test" @@ -483,6 +488,50 @@ def test_credentials_file_rejects_symlinks(tmp_path): assert target.read_text() == "original" +def test_save_waits_for_open_credential_reader(tmp_path, monkeypatch): + path = tmp_path / "user.json" + writer = CredentialStore(path) + writer.lock.timeout = 0 + writer.save(Credentials("original")) + reader = GoogleAuth(settings_file=None) + opened, finish = Event(), Event() + fdopen = os.fdopen + + def hold_reader(fd, mode="r", **kwargs): + stream = fdopen(fd, mode, **kwargs) + if mode == "r": + opened.set() + assert finish.wait(10) + return stream + + monkeypatch.setattr(os, "fdopen", hold_reader) + with ThreadPoolExecutor(max_workers=1) as pool: + loading = pool.submit(reader.LoadCredentialsFile, path) + try: + assert opened.wait(10) + with pytest.raises(Timeout): + writer.save(Credentials("updated")) + finally: + finish.set() + loading.result(timeout=10) + assert reader.credentials.token == "original" + writer.save(Credentials("updated")) + assert writer.load().token == "updated" + + +def test_read_only_and_missing_cache(tmp_path, mocker): + path = tmp_path / "user.json" + store = CredentialStore(path) + acquire = mocker.patch.object( + store.lock, "acquire", side_effect=OSError(errno.EROFS, "read only") + ) + assert store.load() is None + acquire.assert_not_called() + path.write_text('{"token": "provisioned"}') + assert store.load().token == "provisioned" + assert list(tmp_path.iterdir()) == [path] + + def test_reauth_uses_explicit_profile(tmp_path, mocker): default = tmp_path / "default.json" profile = tmp_path / "profile.json" @@ -637,6 +686,50 @@ def request(http, uri, **kwargs): monkeypatch.setattr(httplib2.Http, "request", request) other.Refresh() assert calls == [endpoint] + assert other.client_config == {} + + +@pytest.mark.parametrize("client_file", ["malformed", "service", "other-user"]) +def test_cached_login_ignores_unrelated_client_config( + tmp_path, monkeypatch, client_file +): + monkeypatch.chdir(tmp_path) + path = tmp_path / "user.json" + cached = Credentials( + None, + refresh_token="refresh", + client_id="id", + client_secret="secret", + token_uri="https://cache-only.example.test/token", + ) + path.write_text(cached.to_json()) + config = tmp_path / "client_secrets.json" + if client_file == "malformed": + config.write_text("{") + elif client_file == "service": + config.write_text('{"type":"service_account"}') + else: + config.write_text( + json.dumps( + { + "installed": { + "client_id": "other", + "client_secret": "other-secret", + "token_uri": "https://other.example.test/token", + "auth_uri": "https://other.example.test/auth", + "redirect_uris": ["http://localhost"], + } + } + ) + ) + auth = GoogleAuth(settings_file=None) + auth.LoadCredentialsFile(path) + assert auth.credentials.refresh_token == "refresh" + assert auth.credentials.token_uri == TOKEN_URI + assert auth.client_config == {} + if client_file != "other-user": + with pytest.raises(InvalidConfigError): + auth.LoadClientConfigFile() @pytest.mark.parametrize("expiry", [None, datetime(2099, 1, 1)]) @@ -725,3 +818,22 @@ def check_staged_acl(src, dst): auth.SaveCredentialsFile(path) assert dacl(path) == original assert json.loads(path.read_text())["token"] == "updated" + + +@pytest.mark.skipif(sys.platform != "win32", reason="Windows file compression") +def test_windows_compressed_cache(tmp_path): + import win32file + + path = tmp_path / "user.json" + path.write_bytes(b'{"token":"compressed"}' + b" " * 65536) + subprocess.run( + ["compact.exe", "/C", "/EXE:XPRESS4K", "/F", str(path)], check=True + ) + if ( + not win32file.GetFileAttributes(str(path)) + & stat.FILE_ATTRIBUTE_REPARSE_POINT + ): + pytest.skip("The volume does not support WOF-compressed files") + auth = GoogleAuth(settings_file=None) + auth.LoadCredentialsFile(path) + assert auth.credentials.token == "compressed" From 47d812a01e0851f3d7227b5f73c211e7ffa2164c Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sat, 21 Nov 2026 10:05:58 -0800 Subject: [PATCH 13/15] Verify the compressed cache fixture with the Windows Overlay Filter --- pydrive2/test/test_auth_flows.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/pydrive2/test/test_auth_flows.py b/pydrive2/test/test_auth_flows.py index 3755d4b..3fd7444 100644 --- a/pydrive2/test/test_auth_flows.py +++ b/pydrive2/test/test_auth_flows.py @@ -2,7 +2,6 @@ import json import os import pickle -import stat import subprocess import sys import webbrowser @@ -822,18 +821,26 @@ def check_staged_acl(src, dst): @pytest.mark.skipif(sys.platform != "win32", reason="Windows file compression") def test_windows_compressed_cache(tmp_path): - import win32file + from ctypes import byref, c_int, windll + + import win32api + + if win32api.GetVolumeInformation(tmp_path.anchor)[4] != "NTFS": + pytest.skip("WOF compression requires NTFS") path = tmp_path / "user.json" path.write_bytes(b'{"token":"compressed"}' + b" " * 65536) subprocess.run( ["compact.exe", "/C", "/EXE:XPRESS4K", "/F", str(path)], check=True ) - if ( - not win32file.GetFileAttributes(str(path)) - & stat.FILE_ATTRIBUTE_REPARSE_POINT - ): - pytest.skip("The volume does not support WOF-compressed files") + backed = c_int() + assert ( + windll.wofutil.WofIsExternalFile( + str(path), byref(backed), None, None, None + ) + == 0 + ) + assert backed.value auth = GoogleAuth(settings_file=None) auth.LoadCredentialsFile(path) assert auth.credentials.token == "compressed" From 57640a6a67d07aa7f5e1a14404769749bbb6b260 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sat, 21 Nov 2026 10:11:31 -0800 Subject: [PATCH 14/15] Let Python manage the credential reader descriptor --- pydrive2/_credentials.py | 52 ++++++++++++++++++-------------- pydrive2/test/test_auth_flows.py | 14 +++++---- 2 files changed, 38 insertions(+), 28 deletions(-) diff --git a/pydrive2/_credentials.py b/pydrive2/_credentials.py index 7c5f4d1..60e73b8 100644 --- a/pydrive2/_credentials.py +++ b/pydrive2/_credentials.py @@ -4,7 +4,7 @@ import stat import sys import tempfile -from contextlib import ExitStack +from contextlib import nullcontext from hashlib import sha256 from pathlib import Path from threading import Lock, RLock @@ -212,27 +212,35 @@ def _check_path(path): def _read_file(path, lock): if _check_path(path) is None: return None - with ExitStack() as stack: - try: - stack.enter_context(lock) - except OSError as exc: - # A pre-provisioned cache can be mounted read-only. - if exc.errno not in (errno.EACCES, errno.EPERM, errno.EROFS): - raise - try: - fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) - except FileNotFoundError: - return None - stream = stack.enter_context(os.fdopen(fd, encoding="utf-8")) - if sys.platform == "win32": - # Open normally so Windows can hydrate cloud or compressed files. - # Verify the opened file before reading since Windows lacks O_NOFOLLOW. - info = _check_path(path) - if info is None or not os.path.samestat(info, os.fstat(fd)): - raise InvalidCredentialsError( - "Credentials file changed while opening" - ) - return stream.read() + try: + read_lock = lock.acquire() + except OSError as exc: + # A pre-provisioned cache can be mounted read-only. + if exc.errno not in (errno.EACCES, errno.EPERM, errno.EROFS): + raise + read_lock = nullcontext() + try: + with ( + read_lock, + open(path, encoding="utf-8", opener=_open_no_follow) as reader, + ): + if sys.platform == "win32": + # Open normally to hydrate cloud or compressed files. Windows + # lacks O_NOFOLLOW, so verify the opened file before reading. + info = _check_path(path) + if info is None or not os.path.samestat( + info, os.fstat(reader.fileno()) + ): + raise InvalidCredentialsError( + "Credentials file changed while opening" + ) + return reader.read() + except FileNotFoundError: + return None + + +def _open_no_follow(path, flags): + return os.open(path, flags | getattr(os, "O_NOFOLLOW", 0)) def _write_file(path, serialized): diff --git a/pydrive2/test/test_auth_flows.py b/pydrive2/test/test_auth_flows.py index 3fd7444..3cb3202 100644 --- a/pydrive2/test/test_auth_flows.py +++ b/pydrive2/test/test_auth_flows.py @@ -7,6 +7,7 @@ import webbrowser from base64 import urlsafe_b64encode from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager from datetime import datetime from hashlib import sha256 from http.client import HTTPConnection @@ -494,16 +495,17 @@ def test_save_waits_for_open_credential_reader(tmp_path, monkeypatch): writer.save(Credentials("original")) reader = GoogleAuth(settings_file=None) opened, finish = Event(), Event() - fdopen = os.fdopen - def hold_reader(fd, mode="r", **kwargs): - stream = fdopen(fd, mode, **kwargs) - if mode == "r": + @contextmanager + def hold_reader(*args, **kwargs): + with open(*args, **kwargs) as stream: opened.set() assert finish.wait(10) - return stream + yield stream - monkeypatch.setattr(os, "fdopen", hold_reader) + monkeypatch.setattr( + "pydrive2._credentials.open", hold_reader, raising=False + ) with ThreadPoolExecutor(max_workers=1) as pool: loading = pool.submit(reader.LoadCredentialsFile, path) try: From ebe80fc05ea72536934c9813793e8ce1f5095c14 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sat, 21 Nov 2026 10:35:41 -0800 Subject: [PATCH 15/15] Rebuild authorized Drive transports when switching credentials --- pydrive2/auth.py | 47 +++++++++++++++-------- pydrive2/files.py | 2 +- pydrive2/test/test_auth_flows.py | 64 ++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 16 deletions(-) diff --git a/pydrive2/auth.py b/pydrive2/auth.py index 1af395d..d53a2f1 100644 --- a/pydrive2/auth.py +++ b/pydrive2/auth.py @@ -63,7 +63,10 @@ def _decorated(self, *args, **kwargs): self.auth.credentials, user_credentials.Credentials ): self.auth._refresh_if_expired() - elif self.auth.auth_method == "service": + elif ( + self.auth.credentials is None + and self.auth.auth_method == "service" + ): self.auth.ServiceAuth() else: self.auth.LocalWebserverAuth() @@ -124,7 +127,7 @@ class GoogleAuth(ApiAttributeMixin): settings = ApiAttribute("settings") client_config = ApiAttribute("client_config") flow = ApiAttribute("flow") - credentials = ApiAttribute("credentials") + _credentials = ApiAttribute("credentials") http = ApiAttribute("http") service = ApiAttribute("service") auth_method = ApiAttribute("auth_method") @@ -157,6 +160,19 @@ def __init__( self._configured_store = self._storage_from_settings() self._login_store = self._configured_store + @property + def credentials(self): + return self._credentials + + @credentials.setter + def credentials(self, value): + with self._refresh_lock: + if self._credentials is not value: + self.service = None + if isinstance(self.http, AuthorizedHttp): + self.http = self.http.http + self._credentials = value + @property def access_token_expired(self): """Checks if access token doesn't exist or is expired. @@ -301,7 +317,6 @@ def ServiceAuth(self): info, **options ) ) - self.service = None self._refresh_if_expired() if self.service is None: self.Authorize() @@ -390,6 +405,8 @@ def _load_credentials(self, store): credentials.use_client_config(config) self.credentials = credentials self._login_store = store + # A missing user cache starts a user login, even after ServiceAuth. + self.auth_method = None def SaveCredentials(self, backend=None): """Saves credentials according to specified backend. @@ -653,20 +670,20 @@ def Authorize(self): :raises: AuthenticationError """ - if self.access_token_expired: - raise AuthenticationError( - "No valid credentials provided to authorize" + with self._refresh_lock: + if self.access_token_expired: + raise AuthenticationError( + "No valid credentials provided to authorize" + ) + if self.http is None: + self.http = self._build_http() + if isinstance(self.http, AuthorizedHttp): + self.http = self.http.http + self.http = AuthorizedHttp(self.credentials, http=self.http) + self.service = build( + "drive", "v2", http=self.http, cache_discovery=False ) - if self.http is None: - self.http = self._build_http() - if isinstance(self.http, AuthorizedHttp): - self.http = self.http.http - self.http = AuthorizedHttp(self.credentials, http=self.http) - self.service = build( - "drive", "v2", http=self.http, cache_discovery=False - ) - def Get_Http_Object(self): """Create and authorize an httplib2.Http object. Necessary for thread-safety. diff --git a/pydrive2/files.py b/pydrive2/files.py index a611719..0dddceb 100644 --- a/pydrive2/files.py +++ b/pydrive2/files.py @@ -908,7 +908,7 @@ def _DeletePermission(self, permission_id): try: self.auth.service.permissions().delete( fileId=file_id, permissionId=permission_id - ).execute() + ).execute(http=self.http) except errors.HttpError as error: raise ApiRequestError(error) else: diff --git a/pydrive2/test/test_auth_flows.py b/pydrive2/test/test_auth_flows.py index 3cb3202..139fac9 100644 --- a/pydrive2/test/test_auth_flows.py +++ b/pydrive2/test/test_auth_flows.py @@ -31,6 +31,7 @@ GoogleAuth, InvalidCredentialsError, ) +from pydrive2.drive import GoogleDrive from pydrive2.fs.spec import GDriveAuthError, _client_auth from pydrive2.settings import InvalidConfigError @@ -533,6 +534,69 @@ def test_read_only_and_missing_cache(tmp_path, mocker): assert list(tmp_path.iterdir()) == [path] +@pytest.mark.parametrize("source", ["file", "assignment"]) +def test_switching_credentials_rebuilds_transports(source, tmp_path, mocker): + auth = GoogleAuth(settings_file=None) + raw_http = httplib2.Http() + auth.http = raw_http + first = Credentials("first") + auth.credentials = first + drive = GoogleDrive(auth) + execute = mocker.patch( + "googleapiclient.http.HttpRequest.execute", + autospec=True, + return_value={}, + ) + drive.GetAbout() + old_service = auth.service + old_thread_http = drive.http + auth.credentials = first + assert auth.service is old_service + + second = Credentials("second") + if source == "file": + path = tmp_path / "second.json" + path.write_text(second.to_json()) + auth.LoadCredentialsFile(path) + else: + auth.credentials = second + assert auth.service is None + file = drive.CreateFile({"id": "test-file"}) + file.DeletePermission("test-permission") + request = execute.call_args.args[0] + assert auth.service is not old_service + assert request.http is auth.http + assert request.http.http is raw_http + assert request.http.credentials is auth.credentials + assert execute.call_args.kwargs["http"] is file.http + assert file.http is not old_thread_http + assert file.http.credentials is auth.credentials + assert auth.credentials.token == "second" + + +def test_expired_user_credentials_after_service_login( + service_info, token_endpoint, mocker +): + auth = GoogleAuth( + settings={ + "client_config_backend": "service", + "service_config": {"client_json_dict": service_info}, + } + ) + auth.ServiceAuth() + auth.credentials = Credentials( + None, + refresh_token="user-refresh", + token_uri=TOKEN_URI, + client_id="id", + client_secret="secret", + ) + mocker.patch("googleapiclient.http.HttpRequest.execute", return_value={}) + GoogleDrive(auth).GetAbout() + assert token_endpoint[-1]["refresh_token"] == ["user-refresh"] + assert auth.http.credentials is auth.credentials + + def test_reauth_uses_explicit_profile(tmp_path, mocker): default = tmp_path / "default.json" profile = tmp_path / "profile.json"