diff --git a/docs/fsspec.rst b/docs/fsspec.rst index 516abeb..cbbdd9b 100644 --- a/docs/fsspec.rst +++ b/docs/fsspec.rst @@ -11,6 +11,36 @@ Installation pip install 'pydrive2[fsspec]' +PyDrive2 registers the ``pydrive2://`` protocol with fsspec. The first +component is a case-sensitive Drive ID or one of the aliases ``root`` and +``appDataFolder``. Open an existing file by ID, or name a file relative to a +folder that the account can access: + +.. code-block:: python + + import fsspec + + options = { + "use_service_account": True, + "client_json_file_path": "/path/to/service-account.json", + } + with fsspec.open("pydrive2://FILE_ID", "rb", **options) as stream: + data = stream.read() + + with fsspec.open("pydrive2://FOLDER_ID/report.csv", "rt", **options) as stream: + header = stream.readline() + + fs = fsspec.filesystem("pydrive2", path="FOLDER_ID", **options) + names = fs.ls("FOLDER_ID") + +A filesystem instance is scoped to one root. Use URLs under the same folder +ID or root alias in a single ``fsspec.open_files`` call. Sibling URLs reuse +the filesystem and its directory ID cache. Reads by file ID skip name +lookups. Reads are sequential. To create a file, use a folder ID or root +alias followed by a filename. As with the direct PyDrive2 filesystem, +a write creates a Drive item even when a sibling has the same name. All +reads require credentials, including files shared by link. + Local webserver --------------- diff --git a/pydrive2/fs/spec.py b/pydrive2/fs/spec.py index 6166662..3164010 100644 --- a/pydrive2/fs/spec.py +++ b/pydrive2/fs/spec.py @@ -191,9 +191,11 @@ def _service_auth( class GDriveFileSystem(AbstractFileSystem): """Access to gdrive as an fsspec filesystem""" + protocol = "pydrive2" + def __init__( self, - path, + path="root", google_auth=None, client_id=None, client_secret=None, @@ -208,7 +210,8 @@ def __init__( ): """Create an instance of GDriveFileSystem. - :param path: gdrive path. + :param path: gdrive path. Defaults to ``"root"`` (the Drive + root), so ``fsspec`` can build the filesystem without one. :type path: str. :param google_auth: Authenticated GoogleAuth instance. :type google_auth: GoogleAuth. @@ -271,12 +274,24 @@ def __init__( self._trash_only = trash_only self._acknowledge_abuse = acknowledge_abuse + @classmethod + def _get_kwargs_from_urls(cls, path): + # A filesystem caches names relative to a single Drive root. + root, _, _ = cls._strip_protocol(path).partition("/") + return {} if root in {"", "root"} else {"path": root} + def split_path(self, path): - parts = path.replace("//", "/").rstrip("/").split("/", 1) - if len(parts) == 2: - return parts - else: - return parts[0], "" + path = self._strip_protocol(path).replace("//", "/") + root, _, base = path.partition("/") + return root or "root", base + + def _split_mounted_path(self, path): + root, base = self.split_path(path) + if root != self.root: + raise ValueError( + f"Path {path!r} is outside Drive root {self.root!r}" + ) + return root, base @wrap_prop(threading.RLock()) @cached_property @@ -414,8 +429,7 @@ def _path_to_item_ids(self, path, create=False, use_cache=True): ) def _get_item_id(self, path, create=False, use_cache=True): - bucket, base = self.split_path(path) - assert bucket == self.root + _, base = self._split_mounted_path(path) item_ids = self._path_to_item_ids(base, create, use_cache) if item_ids: @@ -448,8 +462,18 @@ def _create_dir(self, parent_id, name, remote_path): return item["id"] def exists(self, path): + from pydrive2.files import ApiRequestError + + root, base = self._split_mounted_path(path) try: - self._get_item_id(path) + if not base and root not in {"root", "appDataFolder"}: + self.info(path) + else: + self._get_item_id(path) + except ApiRequestError as exc: + if exc.error.get("code") == 404: + return False + raise except FileNotFoundError: return False else: @@ -463,12 +487,11 @@ def info(self, path): gdrive_file.FetchMetadata(fields=_INFO_FIELDS) return _file_entry( - posixpath.join(bucket, base.rstrip("/")), gdrive_file + posixpath.join(bucket, base).rstrip("/"), gdrive_file ) def ls(self, path, detail=False): - bucket, base = self.split_path(path) - assert bucket == self.root + bucket, base = self._split_mounted_path(path) dir_ids = self._path_to_item_ids(base) if not dir_ids: @@ -488,8 +511,7 @@ def ls(self, path, detail=False): return [content["name"] for content in contents] def find(self, path, detail=False, **kwargs): - bucket, base = self.split_path(path) - assert bucket == self.root + bucket, base = self._split_mounted_path(path) # Make sure the base path is cached and dir_ids below has some # dirs revelant to this call @@ -641,6 +663,10 @@ def cb(value, total): def _open(self, path, mode, **kwargs): assert mode in {"rb", "wb"} if mode == "wb": + if not self._split_mounted_path(path)[1]: + raise ValueError( + "To create a Drive file, specify a folder ID and name" + ) return GDriveBufferedWriter(self, path) else: item_id = self._get_item_id(path) diff --git a/pydrive2/test/test_fs.py b/pydrive2/test/test_fs.py index e013fa8..33029fa 100644 --- a/pydrive2/test/test_fs.py +++ b/pydrive2/test/test_fs.py @@ -327,6 +327,16 @@ def test_open_rw(fs, remote_dir): assert stream.read() == data +def test_fsspec_urls(fs, remote_dir): + options = {"google_auth": fs.client.auth} + named_url = f"pydrive2://{remote_dir}/report.txt" + with fsspec.open(named_url, "wt", **options) as stream: + stream.write("hello") + file_id = fs._get_item_id(remote_dir + "/report.txt") + with fsspec.open(f"pydrive2://{file_id}", "rt", **options) as stream: + assert stream.read() == "hello" + + def test_concurrent_operations(fs, fs_factory, remote_dir): # Include an extra dir name to force upload operations creating it # this way we can also test that only a single directory is created diff --git a/pydrive2/test/test_fs_registration.py b/pydrive2/test/test_fs_registration.py new file mode 100644 index 0000000..58a8961 --- /dev/null +++ b/pydrive2/test/test_fs_registration.py @@ -0,0 +1,96 @@ +import subprocess +import sys +from urllib.parse import urlsplit + +import fsspec +import pytest + +from pydrive2.fs import GDriveFileSystem +from pydrive2.fs.spec import FOLDER_MIME_TYPE +from pydrive2.test.http_mock import error_response, query, response + + +def test_fsspec_discovers_backend_without_eager_import(): + subprocess.run( + [ + sys.executable, + "-c", + ( + "import fsspec, sys\n" + "assert 'pydrive2.fs' not in sys.modules\n" + "assert 'googleapiclient.discovery' not in sys.modules\n" + "cls = fsspec.get_filesystem_class('pydrive2')\n" + "from pydrive2.fs import GDriveFileSystem\n" + "assert cls is GDriveFileSystem" + ), + ], + check=True, + ) + + +def test_filesystem_factory(mock_drive): + with pytest.raises(ValueError, match="Specify credentials"): + fsspec.filesystem("pydrive2") + drive, requests = mock_drive() + fs = fsspec.filesystem("pydrive2", google_auth=drive.auth) + assert (fs.root, fs.base) == ("root", "") + mounted = GDriveFileSystem("FoLdEr/sub", google_auth=drive.auth) + assert (mounted.root, mounted.base) == ("FoLdEr", "sub") + assert not requests + + +def test_open_by_file_id(mock_drive): + drive, requests = mock_drive(({"status": "200"}, b"contents")) + with fsspec.open("pydrive2://FiLe_ID", google_auth=drive.auth) as stream: + assert stream.read() == b"contents" + (download,) = requests + assert urlsplit(download[0]).path == "/drive/v3/files/FiLe_ID" + assert query(download)["alt"] == ["media"] + + +def test_open_files_reuses_folder_lookup(mock_drive): + def entry(name, item_id, mime_type="text/plain"): + return response( + {"files": [{"name": name, "id": item_id, "mimeType": mime_type}]} + ) + + drive, requests = mock_drive( + response({"driveId": "shared"}), + entry("sub", "sub-ID", FOLDER_MIME_TYPE), + entry("a", "file-a"), + ({"status": "200"}, b"one"), + entry("b", "file-b"), + ({"status": "200"}, b"two"), + response({"files": []}), + ) + prefix = "pydrive2://FoLdEr/sub" + options = {"google_auth": drive.auth} + first, second = fsspec.open_files( + [prefix + "/a", prefix + "/b"], **options + ) + assert fsspec.open(prefix + "/b", **options).fs is first.fs is second.fs + assert not requests + with first as a, second as b: + assert (a.read(), b.read()) == (b"one", b"two") + assert first.fs.ls(prefix) == [] + shared, sub, a_lookup, _, b_lookup, _, listing = requests + assert urlsplit(shared[0]).path.endswith("/FoLdEr") + assert "'FoLdEr' in parents" in query(sub)["q"][0] + assert all( + "'sub-ID' in parents" in query(r)["q"][0] + for r in (a_lookup, b_lookup, listing) + ) + assert query(listing)["driveId"] == ["shared"] + + +def test_missing_file_id(mock_drive): + drive, requests = mock_drive(error_response(404, "notFound")) + fs, path = fsspec.core.url_to_fs( + "pydrive2://Missing", google_auth=drive.auth + ) + assert not fs.exists(path) + assert len(requests) == 1 + with pytest.raises(ValueError, match="folder ID and name"): + fs.open(path, "wb") + with pytest.raises(ValueError, match="outside Drive root"): + fs.ls("pydrive2://Other/sub") diff --git a/pyproject.toml b/pyproject.toml index 71886cc..95d137b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,9 @@ typing = ["mypy==2.4.1", "types-httplib2", "types-PyYAML"] hook-dirs = "pydrive2.__pyinstaller:get_hook_dirs" tests = "pydrive2.__pyinstaller:get_PyInstaller_tests" +[project.entry-points."fsspec.specs"] +pydrive2 = "pydrive2.fs:GDriveFileSystem" + [tool.setuptools] packages=[ "pydrive2",