From 359a283a3e715ddc6061aa346f577f6738e924ec Mon Sep 17 00:00:00 2001 From: siddhir <13107871+siddhir@users.noreply.github.com> Date: Sun, 15 Nov 2026 11:05:20 -0600 Subject: [PATCH] Add METTLE 1.5 encoder Adds METTLE 1.5 (https://huggingface.co/slideflow-labs/mettle-1.5) as `mettle15`: config, download entry, loader, embedding extraction (1536-dim refined CLS for classification, 1536-dim patch tokens for segmentation), LoRA block names, docs. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01SxonfkyZzXXwd2ggTQZXxQ --- README.md | 1 + docs/index.md | 1 + src/thunder/benchmark.py | 2 +- .../config/pretrained_model/mettle15.yaml | 6 + src/thunder/models/adapters.py | 3 + src/thunder/models/download.py | 12 + src/thunder/models/pretrained_models.py | 42 ++++ src/thunder/utils/constants.py | 1 + src/thunder_bench.egg-info/PKG-INFO | 236 ++++++++++++++++++ src/thunder_bench.egg-info/SOURCES.txt | 154 ++++++++++++ .../dependency_links.txt | 1 + src/thunder_bench.egg-info/entry_points.txt | 2 + src/thunder_bench.egg-info/requires.txt | 35 +++ src/thunder_bench.egg-info/top_level.txt | 1 + 14 files changed, 496 insertions(+), 1 deletion(-) create mode 100644 src/thunder/config/pretrained_model/mettle15.yaml create mode 100644 src/thunder_bench.egg-info/PKG-INFO create mode 100644 src/thunder_bench.egg-info/SOURCES.txt create mode 100644 src/thunder_bench.egg-info/dependency_links.txt create mode 100644 src/thunder_bench.egg-info/entry_points.txt create mode 100644 src/thunder_bench.egg-info/requires.txt create mode 100644 src/thunder_bench.egg-info/top_level.txt diff --git a/README.md b/README.md index d6773e8..19fcca6 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ An API and command line interface (CLI) are provided to allow users to download * OpenMidnight-1.1: https://huggingface.co/medarc/OpenMidnight-1.1 * OpenMidnight-1.1-SPECTRA: https://huggingface.co/medarc/OpenMidnight-1.1-SPECTRA * H-optimus-0-SPECTRA: https://huggingface.co/medarc/H-optimus-0-SPECTRA +* METTLE 1.5: https://huggingface.co/slideflow-labs/mettle-1.5 * Virchow-SPECTRA: https://huggingface.co/medarc/Virchow-SPECTRA * Midnight-12k-SPECTRA: https://huggingface.co/medarc/Midnight-12k-SPECTRA * GenBio-PathFM: https://huggingface.co/genbio-ai/genbio-pathfm diff --git a/docs/index.md b/docs/index.md index 6a92df5..c0db974 100644 --- a/docs/index.md +++ b/docs/index.md @@ -119,6 +119,7 @@ pip install . # install the package |HIBOU-B | hiboub | ViT-B/14 | 86M | DINOv2 | |x | |HIBOU-L | hiboul | ViT-L/14 | 307M | DINOv2 | |x | |H−OPTIMUS−0 | hoptimus0 | ViT-G/14 | 1.1B | DINOv2 | |x | +|METTLE 1.5 | mettle15 | ViT-G/14 | 1.1B | Fine-tuning (base: H−OPTIMUS−0) | |x | |H0-mini | h0mini | ViT-B/14 | 86M | DINOv2 (teacher: H−OPTIMUS−0) | |x | |H−OPTIMUS−1 | hoptimus1 | ViT-G/14 | 1.1B | DINOv2 | |x | |KAIKO-S/8 | kaiko_vits8 | ViT-S/8 | 22M | DINO | |x | diff --git a/src/thunder/benchmark.py b/src/thunder/benchmark.py index 448eb8a..311133f 100644 --- a/src/thunder/benchmark.py +++ b/src/thunder/benchmark.py @@ -21,7 +21,7 @@ def benchmark( where options are: - dataset: *bach*, *bracs*, *break_his*, *ccrcc*, *crc*, *esca*, *mhist*, *ocelot*, *pannuke*, *patch_camelyon*, *segpath_epithelial*, *segpath_lymphocytes*, *tcga_crc_msi*, *tcga_tils*, *tcga_uniform*, *wilds* - - model: *hiboub*, *hiboul*, *hoptimus0*, *hoptimus1*, *genbio-pathfm*, *openmidnight11*, *openmidnight11_spectra*, *hoptimus0_spectra*, *midnight12k_spectra*, *virchow_spectra*, *midnight*, *phikon*, *phikon2*, *uni*, *uni2h*, *virchow*, *virchow2*, *conch*, *titan*, *keep*, *musk*, *plip*, *quiltnetb32*, *dinov2base*, *dinov2large*, *vitbasepatch16224in21k*, *vitlargepatch16224in21k*, *clipvitbasepatch32*, *clipvitlargepatch14* + - model: *hiboub*, *hiboul*, *hoptimus0*, *hoptimus1*, *genbio-pathfm*, *openmidnight11*, *openmidnight11_spectra*, *hoptimus0_spectra*, *mettle15*, *midnight12k_spectra*, *virchow_spectra*, *midnight*, *phikon*, *phikon2*, *uni*, *uni2h*, *virchow*, *virchow2*, *conch*, *titan*, *keep*, *musk*, *plip*, *quiltnetb32*, *dinov2base*, *dinov2large*, *vitbasepatch16224in21k*, *vitlargepatch16224in21k*, *clipvitbasepatch32*, *clipvitlargepatch14* - task: *adversarial_attack*, *alignment_scoring*, *image_retrieval*, *knn*, *linear_probing*, *pre_computing_embeddings*, *segmentation*, *simple_shot*, *transformation_invariance*, *zero_shot_vlm* - loading_mode: *online_loading*, *image_pre_loading*, *embedding_pre_loading* diff --git a/src/thunder/config/pretrained_model/mettle15.yaml b/src/thunder/config/pretrained_model/mettle15.yaml new file mode 100644 index 0000000..f29b0a7 --- /dev/null +++ b/src/thunder/config/pretrained_model/mettle15.yaml @@ -0,0 +1,6 @@ +model_name: mettle15 +type: safetensors +vlm: false +emb_dim: 1536 +ckpt_path: ${oc.env:THUNDER_BASE_DATA_FOLDER}/pretrained_ckpts/mettle1.5/ +hf_tag: hf-hub:slideflow-labs/mettle-1.5 diff --git a/src/thunder/models/adapters.py b/src/thunder/models/adapters.py index 3a910af..6312e42 100644 --- a/src/thunder/models/adapters.py +++ b/src/thunder/models/adapters.py @@ -39,6 +39,9 @@ def get_model_lora_names( elif model_name == "conch": block_name = "visual.trunk.blocks" qkv_name = "attn.qkv" + elif model_name == "mettle15": + block_name = "backbone.blocks" + qkv_name = "attn.qkv" else: raise NotImplementedError( f"LoRA not implemented for {model_name} and {model_type} model type." diff --git a/src/thunder/models/download.py b/src/thunder/models/download.py index b343f0e..9981f5a 100644 --- a/src/thunder/models/download.py +++ b/src/thunder/models/download.py @@ -34,6 +34,10 @@ "medarc/H-optimus-0-SPECTRA", "model.safetensors", ), # H-optimus-0-SPECTRA (https://huggingface.co/medarc/H-optimus-0-SPECTRA) + "mettle15": ( + "slideflow-labs/mettle-1.5", + "model.safetensors", + ), # METTLE 1.5 (https://huggingface.co/slideflow-labs/mettle-1.5) "hoptimus0": ( "bioptimus/H-optimus-0", "pytorch_model.bin", @@ -180,6 +184,7 @@ def download_models(models: Union[List[str], str]) -> None: * virchow2 * hoptimus0 * hoptimus0_spectra + * mettle15 * h0mini * hoptimus1 * genbio-pathfm @@ -296,6 +301,13 @@ def download_model(model: str) -> None: "virchowspectra", ]: extra_files = ["config.json"] + elif local_dir_tag == "mettle1.5": + extra_files = [ + "config.json", + "preprocessor_config.json", + "configuration_mettle.py", + "modeling_mettle.py", + ] elif local_dir_tag == "titan": extra_files = [ "text_transformer.py", diff --git a/src/thunder/models/pretrained_models.py b/src/thunder/models/pretrained_models.py index d25d54e..b25acb2 100644 --- a/src/thunder/models/pretrained_models.py +++ b/src/thunder/models/pretrained_models.py @@ -130,6 +130,8 @@ def get_model(model_cfg: dict, device: str): model, transform = get_midnight(model_cfg.ckpt_path) elif model_cfg.model_name == "genbio-pathfm": model, transform = get_genbio_pathfm(model_cfg.ckpt_path) + elif model_cfg.model_name == "mettle15": + model, transform = get_mettle(model_cfg.ckpt_path) else: model, transform, tokenizer = get_from_safetensors( model_cfg.ckpt_path, use_fast="dinov3" in model_cfg.model_name @@ -348,6 +350,17 @@ def extract_embedding(src, pretrained_model, task_type): else: return patch_tokens + elif model_cfg.model_name == "mettle15": + + def extract_embedding(src, pretrained_model, task_type="linear_probing"): + out = pretrained_model(src) + if task_type != "segmentation": + # Refined CLS embedding (the model's default feature view). + return out.pooler_output + else: + # Spatial patch tokens, ignoring CLS and the 4 register tokens. + return out.last_hidden_state[:, 5:] + elif model_cfg.model_name in ["midnight", "midnight12k_spectra"]: def extract_embedding(src, pretrained_model, task_type): @@ -406,6 +419,7 @@ def get_model_from_name(model_name: str, device: str): * virchow2 * hoptimus0 * hoptimus0_spectra + * mettle15 * h0mini * hoptimus1 * genbio-pathfm @@ -819,3 +833,31 @@ def get_genbio_pathfm(ckpt_path: str): return model, transform + +def get_mettle(ckpt_path: str): + """ + Adapted from: + - https://huggingface.co/slideflow-labs/mettle-1.5 + + :param ckpt_path: path to the folder with the stored checkpoint and model code. + """ + from torchvision import transforms + + # Model + model = AutoModel.from_pretrained(ckpt_path, trust_remote_code=True) + + # Transform (as specified in the model's preprocessor_config.json) + transform = transforms.Compose( + [ + transforms.Resize( + (224, 224), interpolation=transforms.InterpolationMode.BICUBIC + ), + transforms.ToTensor(), + transforms.Normalize( + mean=(0.707223, 0.578729, 0.703617), + std=(0.211883, 0.230117, 0.177517), + ), + ] + ) + + return model, transform diff --git a/src/thunder/utils/constants.py b/src/thunder/utils/constants.py index 849ae27..07869fc 100644 --- a/src/thunder/utils/constants.py +++ b/src/thunder/utils/constants.py @@ -8,6 +8,7 @@ class ModelConstants(Enum): "hiboul", "hoptimus0", "hoptimus0_spectra", + "mettle15", "h0mini", "hoptimus1", "genbio-pathfm", diff --git a/src/thunder_bench.egg-info/PKG-INFO b/src/thunder_bench.egg-info/PKG-INFO new file mode 100644 index 0000000..ae2b25b --- /dev/null +++ b/src/thunder_bench.egg-info/PKG-INFO @@ -0,0 +1,236 @@ +Metadata-Version: 2.4 +Name: thunder-bench +Version: 0.2.2 +Summary: THUNDER: Tile-level Histopathology image UNDERstanding benchmark +License-Expression: CC-BY-4.0 +Requires-Python: <3.14,>=3.10 +Description-Content-Type: text/markdown +License-File: LICENSE +Requires-Dist: einops>=0.8.1 +Requires-Dist: einops_exts>=0.0.4 +Requires-Dist: h5py>=3.12.1 +Requires-Dist: huggingface-hub>=0.35.3 +Requires-Dist: hydra-core>=1.3.2 +Requires-Dist: ijson>=3.4.0 +Requires-Dist: kornia>=0.8.0 +Requires-Dist: matplotlib>=3.9.2 +Requires-Dist: numpy>=1.26.4 +Requires-Dist: opencv-python>=4.10.0.84 +Requires-Dist: omegaconf>=2.3.0 +Requires-Dist: pandas>=2.2.3 +Requires-Dist: pillow>=11.0.0 +Requires-Dist: plotly>=6.0.0 +Requires-Dist: pydantic<2.12 +Requires-Dist: scikit-learn>=1.6.1 +Requires-Dist: sentencepiece>=0.2.1 +Requires-Dist: timm<=1.0.20,>=1.0.13 +Requires-Dist: torch>=2.6.0 +Requires-Dist: torchaudio>=2.6.0 +Requires-Dist: torchvision>=0.21.0 +Requires-Dist: tqdm>=4.66.5 +Requires-Dist: transformers>=4.57.1 +Requires-Dist: typer>=0.20.0 +Requires-Dist: wandb>=0.19.4 +Requires-Dist: wilds>=2.0.0 +Provides-Extra: dev +Requires-Dist: black>=25.1.0; extra == "dev" +Requires-Dist: isort>=6.0.1; extra == "dev" +Requires-Dist: mkdocs>=1.6.1; extra == "dev" +Requires-Dist: mkdocs-material>=9.6.11; extra == "dev" +Requires-Dist: mkdocstrings>=0.29.1; extra == "dev" +Requires-Dist: mkdocstrings-python>=1.16.10; extra == "dev" +Requires-Dist: pytest>=8.4.2; extra == "dev" +Dynamic: license-file + + +# Tile-level Histopathology image Understanding benchmark + + + +
+ +[![Paper](https://img.shields.io/badge/THUNDER-arXiv.2507.07860-purple.svg)](https://arxiv.org/abs/2507.07860) +[![PyPI](https://img.shields.io/pypi/v/thunder-bench.svg)](https://pypi.org/project/thunder-bench/) +[![Python application](https://github.com/MICS-lab/thunder/actions/workflows/ci.yml/badge.svg)](https://github.com/MICS-lab/thunder/actions/workflows/ci.yml) +[![Docs](https://img.shields.io/badge/docs-mkdocs-blue)](https://mics-lab.github.io/thunder/) +[![License](https://img.shields.io/badge/License-CC%20BY%204.0-lightgrey.svg)](https://github.com/MICS-Lab/thunder/blob/main/LICENSE) +[![Black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://black.readthedocs.io/en/stable/) + +
+ +
+ +**Published at NeurIPS 2025 Datasets and Benchmarks Track (Spotlight)** + +
+ +We introduce **THUNDER**, a comprehensive benchmark designed to rigorously compare foundation models across various downstream tasks in computational pathology. THUNDER enables the evaluation and analysis of feature representations, robustness, and uncertainty quantification of these models across different datasets. Our benchmark encompasses a diverse collection of well-established datasets, covering multiple cancer types, image magnifications, and varying image and sample sizes. We propose an extensive set of tasks aimed at thoroughly assessing the capabilities and limitations of foundation models in digital pathology. + +⚡ **Paper**: [THUNDER: Tile-level Histopathology image UNDERstanding benchmark](https://arxiv.org/abs/2507.07860)\ +⚡ **Homepage/Documentation**: [THUNDER docs](https://mics-lab.github.io/thunder/)\ +⚡ **Examples**: [THUNDER examples](https://mics-lab.github.io/thunder/examples/)\ +⚡ **Reproduce results**: [THUNDER reproduction guidelines](https://mics-lab.github.io/thunder/guidelines/)\ +⚡ **Leaderboards**: [THUNDER leaderboards](https://mics-lab.github.io/thunder/leaderboards/) + +## News + +* **2026-04-08**: GenBio-PathFM is now supported in THUNDER. Its downstream performance was integrated into the [up-to-date rank-sum leaderboard](https://mics-lab.github.io/thunder/leaderboards/#up-to-date-rank-sum-leaderboard) and the [SPIDER leaderboard](https://mics-lab.github.io/thunder/leaderboards/#spider-leaderboard). +* **2025-12-07**: OpenMidnight is now supported in THUNDER. Its downstream performance was integrated into the [up-to-date rank-sum leaderboard](https://mics-lab.github.io/thunder/leaderboards/#up-to-date-rank-sum-leaderboard) and the [SPIDER leaderboard](https://mics-lab.github.io/thunder/leaderboards/#spider-leaderboard). +* **2025-11-21**: As requested (https://github.com/MICS-Lab/thunder/issues/6), H0-mini is now supported in THUNDER. Its downstream performance was integrated into the [up-to-date rank-sum leaderboard](https://mics-lab.github.io/thunder/leaderboards/#up-to-date-rank-sum-leaderboard) and the [SPIDER leaderboard](https://mics-lab.github.io/thunder/leaderboards/#spider-leaderboard). +* **2025-11-14**: GIGAPATH and KAIKO-ViT model variants (S/8, S/16, B/8, B/16) are now supported in THUNDER. The downstream performance of GIGAPATH, KAIKO-S/16 and KAIKO-B/16 were integrated into the [up-to-date rank-sum leaderboard](https://mics-lab.github.io/thunder/leaderboards/#up-to-date-rank-sum-leaderboard) and the [SPIDER leaderboard](https://mics-lab.github.io/thunder/leaderboards/#spider-leaderboard). +* **2025-11-11**: You can now run any task in THUNDER on a custom dataset. Check the [tutorial](https://mics-lab.github.io/thunder/custom_dataset/) to learn more about this feature. +* **2025-10-30**: DINOv3 model variants (B, S, L) are now supported in THUNDER. Their downstream performance was integrated along with SPIDER results into a new [up-to-date rank-sum leaderboard](https://mics-lab.github.io/thunder/leaderboards/#up-to-date-rank-sum-leaderboard). +* **2025-10-06**: As requested (https://github.com/MICS-Lab/thunder/issues/1), a new *zero-shot classification* task to evaluate VLMs was included into THUNDER. Example command: `thunder benchmark keep spider_breast zero_shot_vlm`. See the dedicated [zero-shot classification leaderboard](https://mics-lab.github.io/thunder/leaderboards/#zero-shot-vlm-classification-leaderboard). +* **2025-09-30**: Patch-level SPIDER datasets have been integrated into THUNDER. See the dedicated [SPIDER leaderboard](https://mics-lab.github.io/thunder/leaderboards/#spider-leaderboard). +* **2025-09-18**: THUNDER was accepted to **NeurIPS 2025 Datasets & Benchmarks Track** as a **Spotlight** presentation! + + +## Overview + +We propose a benchmark to compare and study foundation models across three axes: (i) downstream task performance, (ii) feature space comparisons, and (iii) uncertainty and robustness. Our current version integrates 23 foundation models, vision-only, vision-language, trained on pathology or natural images, on 16 datasets covering different magnifications and organs. THUNDER also supports the use of new user-defined models for direct comparisons. + + + + +## Usage +To learn more about how to use `thunder`, please visit our [documentation](https://mics-lab.github.io/thunder/). + +An API and command line interface (CLI) are provided to allow users to download datasets, models, and run benchmarks. The API is designed to be user-friendly and allows for easy integration into existing workflows. The CLI provides a convenient way to access the same functionality from the command line. + +> [!IMPORTANT] +> **Downloading supported foundation models**: you will have to visit the Huggingface URL of supported models you wish to use in order to accept usage conditions. + +
+List of Huggingface URLs + +* UNI: https://huggingface.co/MahmoodLab/UNI +* UNI2-h: https://huggingface.co/MahmoodLab/UNI2-h +* Virchow: https://huggingface.co/paige-ai/Virchow +* Virchow2: https://huggingface.co/paige-ai/Virchow2 +* H-optimus-0: https://huggingface.co/bioptimus/H-optimus-0 +* H-optimus-1: https://huggingface.co/bioptimus/H-optimus-1 +* OpenMidnight-1.1: https://huggingface.co/medarc/OpenMidnight-1.1 +* OpenMidnight-1.1-SPECTRA: https://huggingface.co/medarc/OpenMidnight-1.1-SPECTRA +* H-optimus-0-SPECTRA: https://huggingface.co/medarc/H-optimus-0-SPECTRA +* METTLE 1.5: https://huggingface.co/slideflow-labs/mettle-1.5 +* Virchow-SPECTRA: https://huggingface.co/medarc/Virchow-SPECTRA +* Midnight-12k-SPECTRA: https://huggingface.co/medarc/Midnight-12k-SPECTRA +* GenBio-PathFM: https://huggingface.co/genbio-ai/genbio-pathfm +* CONCH: https://huggingface.co/MahmoodLab/CONCH +* TITAN/CONCHv1.5: https://huggingface.co/MahmoodLab/TITAN +* Phikon: https://huggingface.co/owkin/phikon +* Phikon2: https://huggingface.co/owkin/phikon-v2 +* Hibou-b: https://huggingface.co/histai/hibou-b +* Hibou-L: https://huggingface.co/histai/hibou-L +* Midnight-12k: https://huggingface.co/kaiko-ai/midnight +* KEEP: https://huggingface.co/Astaxanthin/KEEP +* QuiltNet-B-32: https://huggingface.co/wisdomik/QuiltNet-B-32 +* PLIP: https://huggingface.co/vinid/plip +* MUSK: https://huggingface.co/xiangjx/musk +* DINOv2-B: https://huggingface.co/facebook/dinov2-base +* DINOv2-L: https://huggingface.co/facebook/dinov2-large +* ViT-B: https://huggingface.co/google/vit-base-patch16-224-in21k +* ViT-L: https://huggingface.co/google/vit-large-patch16-224-in21k +* CLIP-B: https://huggingface.co/openai/clip-vit-base-patch32 +* CLIP-L: https://huggingface.co/openai/clip-vit-large-patch14 + +
+ +### API Usage +When using the API you can run the following code to download datasets, models and run a benchmark: + +```python +from thunder import benchmark + +benchmark("phikon", "break_his", "knn") +``` + +### CLI Usage +When using the CLI you can run the following command to see all available options, + +```console +thunder --help +``` + +In order to reproduce the above example you can run the following command: + +```console +thunder benchmark phikon break_his knn +``` + +### Reproducing the leaderboard results (SLURM) + +To reproduce our [leaderboard](https://mics-lab.github.io/thunder/leaderboards/) results for a given model across all datasets and tasks, we provide a SLURM array script: + +```console +sbatch scripts/benchmark_all_tasks.sh +``` + +It also works with a custom model by passing a `custom:` path to your model file: + +```console +sbatch scripts/benchmark_all_tasks.sh custom:/path/to/your/model.py +``` + +Once all array tasks finish, gather the outputs with `thunder results-summary`; the reproduced leaderboard scores are the `benchmark_*` rows of the resulting table. The script is a generic SLURM template, you will likely need to adapt the `#SBATCH` header at the top of the file to your cluster before it will run. The segmentation datasets need a GPU with at least 32 GB of VRAM. See the [guidelines](https://mics-lab.github.io/thunder/guidelines/) for details. + +### Extracting embeddings with any supported foundation model (API Usage) +We also provide a [`get_model_from_name`](https://mics-lab.github.io/thunder/api/#thunder.models.get_model_from_name) function through our API to extract embeddings using any foundation model we support on your own data. Below is an example if you want to get the Pytorch callable, transforms and function to extract embeddings for `uni2h`: + +```python +from thunder.models import get_model_from_name + +model, transform, get_embeddings = get_model_from_name("uni2h", device="cuda") +``` + +## Installing thunder + +Code tested with Python 3.10. To replicate, you can create the following conda environment and activate it, +```console +conda create -n thunder_env python=3.10 +conda activate thunder_env +``` + +To install `thunder` run one of the following commands: + +#### From PyPi +```console +pip install thunder-bench +``` + +#### From Source +```console +pip install -e . # install the package in editable mode +``` +```console +pip install . # install the package +``` + +Before running `thunder`, ensure that the environment variable `THUNDER_BASE_DATA_FOLDER` is defined. This variable specifies the path where outputs, foundation models, and datasets will be stored. You can set it by running: + +```console +export THUNDER_BASE_DATA_FOLDER="/path/to/your/data/folder" +``` + +Replace `/path/to/your/data/folder` with your desired storage directory. + +If you want to use the CONCH and MUSK models, you should install them as follows: + +```console +pip install git+https://github.com/Mahmoodlab/CONCH.git # CONCH +pip install git+https://github.com/lilab-stanford/MUSK.git # MUSK +``` + +## Citation +``` +@article{marza2025thunder, + title={{THUNDER}: Tile-level Histopathology image UNDERstanding benchmark}, + author={Marza, Pierre and Fillioux, Leo and Boutaj, Sofi{\`e}ne and Mahatha, Kunal and Desrosiers, Christian and Piantanida, Pablo and Dolz, Jose and Christodoulidis, Stergios and Vakalopoulou, Maria}, + journal={Neural Information Processing Systems (NeurIPS) D&B Track}, + year={2025} +} +``` + +## Acknowledgments +This work has been partially supported by *ANR-23-IAHU-0002*, *ANR-21-CE45-0007*, *ANR-23-CE45-0029*, and the *Health Data Hub (HDH)* as part of the second edition of the *France-Québec* call for projects *Intelligence Artificielle en santé*. It was performed using computational resources from the *Mésocentre* computing center of *Université Paris-Saclay*, *CentraleSupélec* and *École Normale Supérieure Paris-Saclay* supported by *CNRS* and *Région Île-de-France*, and from *GENCI-IDRIS* +(Grant *2025-AD011016068*). diff --git a/src/thunder_bench.egg-info/SOURCES.txt b/src/thunder_bench.egg-info/SOURCES.txt new file mode 100644 index 0000000..258dbfb --- /dev/null +++ b/src/thunder_bench.egg-info/SOURCES.txt @@ -0,0 +1,154 @@ +LICENSE +README.md +pyproject.toml +src/thunder/__init__.py +src/thunder/benchmark.py +src/thunder/main.py +src/thunder/config/config.yaml +src/thunder/config/adaptation/frozen.yaml +src/thunder/config/adaptation/lora.yaml +src/thunder/config/ckpt_saving/save_best_ckpt_only.yaml +src/thunder/config/ckpt_saving/save_ckpts_all_epochs.yaml +src/thunder/config/data_loading/embedding_pre_loading.yaml +src/thunder/config/data_loading/image_pre_loading.yaml +src/thunder/config/data_loading/online_loading.yaml +src/thunder/config/dataset/bach.yaml +src/thunder/config/dataset/bracs.yaml +src/thunder/config/dataset/break_his.yaml +src/thunder/config/dataset/ccrcc.yaml +src/thunder/config/dataset/crc.yaml +src/thunder/config/dataset/esca.yaml +src/thunder/config/dataset/mhist.yaml +src/thunder/config/dataset/ocelot.yaml +src/thunder/config/dataset/pannuke.yaml +src/thunder/config/dataset/patch_camelyon.yaml +src/thunder/config/dataset/segpath_epithelial.yaml +src/thunder/config/dataset/segpath_lymphocytes.yaml +src/thunder/config/dataset/spider_breast.yaml +src/thunder/config/dataset/spider_colorectal.yaml +src/thunder/config/dataset/spider_skin.yaml +src/thunder/config/dataset/spider_thorax.yaml +src/thunder/config/dataset/starc9.yaml +src/thunder/config/dataset/tcga_crc_msi.yaml +src/thunder/config/dataset/tcga_tils.yaml +src/thunder/config/dataset/tcga_uniform.yaml +src/thunder/config/dataset/wilds.yaml +src/thunder/config/embedding_recomputing/no_recomp_embs.yaml +src/thunder/config/embedding_recomputing/recomp_embs.yaml +src/thunder/config/model_retraining/no_retrain_model.yaml +src/thunder/config/model_retraining/retrain_model.yaml +src/thunder/config/pretrained_model/clipvitbasepatch32.yaml +src/thunder/config/pretrained_model/clipvitlargepatch14.yaml +src/thunder/config/pretrained_model/conch.yaml +src/thunder/config/pretrained_model/dinov2base.yaml +src/thunder/config/pretrained_model/dinov2large.yaml +src/thunder/config/pretrained_model/dinov3vitb16pretrainlvd1689m.yaml +src/thunder/config/pretrained_model/dinov3vitl16pretrainlvd1689m.yaml +src/thunder/config/pretrained_model/dinov3vits16pretrainlvd1689m.yaml +src/thunder/config/pretrained_model/genbio-pathfm.yaml +src/thunder/config/pretrained_model/h0mini.yaml +src/thunder/config/pretrained_model/hiboub.yaml +src/thunder/config/pretrained_model/hiboul.yaml +src/thunder/config/pretrained_model/hoptimus0.yaml +src/thunder/config/pretrained_model/hoptimus0_spectra.yaml +src/thunder/config/pretrained_model/hoptimus1.yaml +src/thunder/config/pretrained_model/kaiko_vitb16.yaml +src/thunder/config/pretrained_model/kaiko_vitb8.yaml +src/thunder/config/pretrained_model/kaiko_vits16.yaml +src/thunder/config/pretrained_model/kaiko_vits8.yaml +src/thunder/config/pretrained_model/keep.yaml +src/thunder/config/pretrained_model/mettle15.yaml +src/thunder/config/pretrained_model/midnight.yaml +src/thunder/config/pretrained_model/midnight12k_spectra.yaml +src/thunder/config/pretrained_model/musk.yaml +src/thunder/config/pretrained_model/openmidnight.yaml +src/thunder/config/pretrained_model/openmidnight11.yaml +src/thunder/config/pretrained_model/openmidnight11_spectra.yaml +src/thunder/config/pretrained_model/phikon.yaml +src/thunder/config/pretrained_model/phikon2.yaml +src/thunder/config/pretrained_model/plip.yaml +src/thunder/config/pretrained_model/provgigapath.yaml +src/thunder/config/pretrained_model/quiltnetb32.yaml +src/thunder/config/pretrained_model/titan.yaml +src/thunder/config/pretrained_model/uni.yaml +src/thunder/config/pretrained_model/uni2h.yaml +src/thunder/config/pretrained_model/virchow.yaml +src/thunder/config/pretrained_model/virchow2.yaml +src/thunder/config/pretrained_model/virchow_spectra.yaml +src/thunder/config/pretrained_model/vitbasepatch16224in21k.yaml +src/thunder/config/pretrained_model/vitlargepatch16224in21k.yaml +src/thunder/config/task/adversarial_attack.yaml +src/thunder/config/task/alignment_scoring.yaml +src/thunder/config/task/image_retrieval.yaml +src/thunder/config/task/knn.yaml +src/thunder/config/task/linear_probing.yaml +src/thunder/config/task/pre_computing_embeddings.yaml +src/thunder/config/task/segmentation.yaml +src/thunder/config/task/simple_shot.yaml +src/thunder/config/task/transformation_invariance.yaml +src/thunder/config/task/zero_shot_vlm.yaml +src/thunder/config/wandb/offline.yaml +src/thunder/config/wandb/online.yaml +src/thunder/datasets/__init__.py +src/thunder/datasets/data_splits.py +src/thunder/datasets/download.py +src/thunder/datasets/utils.py +src/thunder/datasets/dataset/__init__.py +src/thunder/datasets/dataset/bach.py +src/thunder/datasets/dataset/bracs.py +src/thunder/datasets/dataset/break_his.py +src/thunder/datasets/dataset/ccrcc.py +src/thunder/datasets/dataset/crc.py +src/thunder/datasets/dataset/esca.py +src/thunder/datasets/dataset/mhist.py +src/thunder/datasets/dataset/ocelot.py +src/thunder/datasets/dataset/pannuke.py +src/thunder/datasets/dataset/patch_camelyon.py +src/thunder/datasets/dataset/segpath_epithelial.py +src/thunder/datasets/dataset/segpath_lymphocytes.py +src/thunder/datasets/dataset/spider_breast.py +src/thunder/datasets/dataset/spider_colorectal.py +src/thunder/datasets/dataset/spider_skin.py +src/thunder/datasets/dataset/spider_thorax.py +src/thunder/datasets/dataset/starc9.py +src/thunder/datasets/dataset/tcga_crc_msi.py +src/thunder/datasets/dataset/tcga_tils.py +src/thunder/datasets/dataset/tcga_uniform.py +src/thunder/datasets/dataset/wilds.py +src/thunder/models/__init__.py +src/thunder/models/adapters.py +src/thunder/models/download.py +src/thunder/models/pretrained_models.py +src/thunder/models/task_specific_models.py +src/thunder/models/utils.py +src/thunder/tasks/__init__.py +src/thunder/tasks/adversarial_attack.py +src/thunder/tasks/alignment_scoring.py +src/thunder/tasks/image_retrieval.py +src/thunder/tasks/knn_classification.py +src/thunder/tasks/pre_computing_patch_embeddings.py +src/thunder/tasks/simple_shot.py +src/thunder/tasks/train_eval_probe.py +src/thunder/tasks/transformation_invariance.py +src/thunder/tasks/zero_shot_vlm.py +src/thunder/utils/__init__.py +src/thunder/utils/alignment_metrics.py +src/thunder/utils/calibration_metrics.py +src/thunder/utils/config.py +src/thunder/utils/constants.py +src/thunder/utils/data.py +src/thunder/utils/dice_loss.py +src/thunder/utils/downstream_metrics.py +src/thunder/utils/pgd_attack_linear.py +src/thunder/utils/results.py +src/thunder/utils/transforms.py +src/thunder/utils/utils.py +src/thunder_bench.egg-info/PKG-INFO +src/thunder_bench.egg-info/SOURCES.txt +src/thunder_bench.egg-info/dependency_links.txt +src/thunder_bench.egg-info/entry_points.txt +src/thunder_bench.egg-info/requires.txt +src/thunder_bench.egg-info/top_level.txt +tests/test_benchmark.py +tests/test_datasets.py +tests/test_models.py \ No newline at end of file diff --git a/src/thunder_bench.egg-info/dependency_links.txt b/src/thunder_bench.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/thunder_bench.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/src/thunder_bench.egg-info/entry_points.txt b/src/thunder_bench.egg-info/entry_points.txt new file mode 100644 index 0000000..3e9d90a --- /dev/null +++ b/src/thunder_bench.egg-info/entry_points.txt @@ -0,0 +1,2 @@ +[console_scripts] +thunder = thunder.main:app diff --git a/src/thunder_bench.egg-info/requires.txt b/src/thunder_bench.egg-info/requires.txt new file mode 100644 index 0000000..4b20cdc --- /dev/null +++ b/src/thunder_bench.egg-info/requires.txt @@ -0,0 +1,35 @@ +einops>=0.8.1 +einops_exts>=0.0.4 +h5py>=3.12.1 +huggingface-hub>=0.35.3 +hydra-core>=1.3.2 +ijson>=3.4.0 +kornia>=0.8.0 +matplotlib>=3.9.2 +numpy>=1.26.4 +opencv-python>=4.10.0.84 +omegaconf>=2.3.0 +pandas>=2.2.3 +pillow>=11.0.0 +plotly>=6.0.0 +pydantic<2.12 +scikit-learn>=1.6.1 +sentencepiece>=0.2.1 +timm<=1.0.20,>=1.0.13 +torch>=2.6.0 +torchaudio>=2.6.0 +torchvision>=0.21.0 +tqdm>=4.66.5 +transformers>=4.57.1 +typer>=0.20.0 +wandb>=0.19.4 +wilds>=2.0.0 + +[dev] +black>=25.1.0 +isort>=6.0.1 +mkdocs>=1.6.1 +mkdocs-material>=9.6.11 +mkdocstrings>=0.29.1 +mkdocstrings-python>=1.16.10 +pytest>=8.4.2 diff --git a/src/thunder_bench.egg-info/top_level.txt b/src/thunder_bench.egg-info/top_level.txt new file mode 100644 index 0000000..e151af7 --- /dev/null +++ b/src/thunder_bench.egg-info/top_level.txt @@ -0,0 +1 @@ +thunder