Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ An API and command line interface (CLI) are provided to allow users to download
* OpenMidnight-1.1: https://huggingface.co/medarc/OpenMidnight-1.1
* OpenMidnight-1.1-SPECTRA: https://huggingface.co/medarc/OpenMidnight-1.1-SPECTRA
* H-optimus-0-SPECTRA: https://huggingface.co/medarc/H-optimus-0-SPECTRA
* METTLE 1.5: https://huggingface.co/slideflow-labs/mettle-1.5
* Virchow-SPECTRA: https://huggingface.co/medarc/Virchow-SPECTRA
* Midnight-12k-SPECTRA: https://huggingface.co/medarc/Midnight-12k-SPECTRA
* GenBio-PathFM: https://huggingface.co/genbio-ai/genbio-pathfm
Expand Down
1 change: 1 addition & 0 deletions docs/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ pip install . # install the package
|HIBOU-B | hiboub | ViT-B/14 | 86M | DINOv2 | |x |
|HIBOU-L | hiboul | ViT-L/14 | 307M | DINOv2 | |x |
|H−OPTIMUS−0 | hoptimus0 | ViT-G/14 | 1.1B | DINOv2 | |x |
|METTLE 1.5 | mettle15 | ViT-G/14 | 1.1B | Fine-tuning (base: H−OPTIMUS−0) | |x |
|H0-mini | h0mini | ViT-B/14 | 86M | DINOv2 (teacher: H−OPTIMUS−0) | |x |
|H−OPTIMUS−1 | hoptimus1 | ViT-G/14 | 1.1B | DINOv2 | |x |
|KAIKO-S/8 | kaiko_vits8 | ViT-S/8 | 22M | DINO | |x |
Expand Down
2 changes: 1 addition & 1 deletion src/thunder/benchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ def benchmark(

where options are:
- dataset: *bach*, *bracs*, *break_his*, *ccrcc*, *crc*, *esca*, *mhist*, *ocelot*, *pannuke*, *patch_camelyon*, *segpath_epithelial*, *segpath_lymphocytes*, *tcga_crc_msi*, *tcga_tils*, *tcga_uniform*, *wilds*
- model: *hiboub*, *hiboul*, *hoptimus0*, *hoptimus1*, *genbio-pathfm*, *openmidnight11*, *openmidnight11_spectra*, *hoptimus0_spectra*, *midnight12k_spectra*, *virchow_spectra*, *midnight*, *phikon*, *phikon2*, *uni*, *uni2h*, *virchow*, *virchow2*, *conch*, *titan*, *keep*, *musk*, *plip*, *quiltnetb32*, *dinov2base*, *dinov2large*, *vitbasepatch16224in21k*, *vitlargepatch16224in21k*, *clipvitbasepatch32*, *clipvitlargepatch14*
- model: *hiboub*, *hiboul*, *hoptimus0*, *hoptimus1*, *genbio-pathfm*, *openmidnight11*, *openmidnight11_spectra*, *hoptimus0_spectra*, *mettle15*, *midnight12k_spectra*, *virchow_spectra*, *midnight*, *phikon*, *phikon2*, *uni*, *uni2h*, *virchow*, *virchow2*, *conch*, *titan*, *keep*, *musk*, *plip*, *quiltnetb32*, *dinov2base*, *dinov2large*, *vitbasepatch16224in21k*, *vitlargepatch16224in21k*, *clipvitbasepatch32*, *clipvitlargepatch14*
- task: *adversarial_attack*, *alignment_scoring*, *image_retrieval*, *knn*, *linear_probing*, *pre_computing_embeddings*, *segmentation*, *simple_shot*, *transformation_invariance*, *zero_shot_vlm*
- loading_mode: *online_loading*, *image_pre_loading*, *embedding_pre_loading*

Expand Down
6 changes: 6 additions & 0 deletions src/thunder/config/pretrained_model/mettle15.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
model_name: mettle15
type: safetensors
vlm: false
emb_dim: 1536
ckpt_path: ${oc.env:THUNDER_BASE_DATA_FOLDER}/pretrained_ckpts/mettle1.5/
hf_tag: hf-hub:slideflow-labs/mettle-1.5
3 changes: 3 additions & 0 deletions src/thunder/models/adapters.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,9 @@ def get_model_lora_names(
elif model_name == "conch":
block_name = "visual.trunk.blocks"
qkv_name = "attn.qkv"
elif model_name == "mettle15":
block_name = "backbone.blocks"
qkv_name = "attn.qkv"
else:
raise NotImplementedError(
f"LoRA not implemented for {model_name} and {model_type} model type."
Expand Down
12 changes: 12 additions & 0 deletions src/thunder/models/download.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@
"medarc/H-optimus-0-SPECTRA",
"model.safetensors",
), # H-optimus-0-SPECTRA (https://huggingface.co/medarc/H-optimus-0-SPECTRA)
"mettle15": (
"slideflow-labs/mettle-1.5",
"model.safetensors",
), # METTLE 1.5 (https://huggingface.co/slideflow-labs/mettle-1.5)
"hoptimus0": (
"bioptimus/H-optimus-0",
"pytorch_model.bin",
Expand Down Expand Up @@ -180,6 +184,7 @@ def download_models(models: Union[List[str], str]) -> None:
* virchow2
* hoptimus0
* hoptimus0_spectra
* mettle15
* h0mini
* hoptimus1
* genbio-pathfm
Expand Down Expand Up @@ -296,6 +301,13 @@ def download_model(model: str) -> None:
"virchowspectra",
]:
extra_files = ["config.json"]
elif local_dir_tag == "mettle1.5":
extra_files = [
"config.json",
"preprocessor_config.json",
"configuration_mettle.py",
"modeling_mettle.py",
]
elif local_dir_tag == "titan":
extra_files = [
"text_transformer.py",
Expand Down
42 changes: 42 additions & 0 deletions src/thunder/models/pretrained_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,8 @@ def get_model(model_cfg: dict, device: str):
model, transform = get_midnight(model_cfg.ckpt_path)
elif model_cfg.model_name == "genbio-pathfm":
model, transform = get_genbio_pathfm(model_cfg.ckpt_path)
elif model_cfg.model_name == "mettle15":
model, transform = get_mettle(model_cfg.ckpt_path)
else:
model, transform, tokenizer = get_from_safetensors(
model_cfg.ckpt_path, use_fast="dinov3" in model_cfg.model_name
Expand Down Expand Up @@ -348,6 +350,17 @@ def extract_embedding(src, pretrained_model, task_type):
else:
return patch_tokens

elif model_cfg.model_name == "mettle15":

def extract_embedding(src, pretrained_model, task_type="linear_probing"):
out = pretrained_model(src)
if task_type != "segmentation":
# Refined CLS embedding (the model's default feature view).
return out.pooler_output
else:
# Spatial patch tokens, ignoring CLS and the 4 register tokens.
return out.last_hidden_state[:, 5:]

elif model_cfg.model_name in ["midnight", "midnight12k_spectra"]:

def extract_embedding(src, pretrained_model, task_type):
Expand Down Expand Up @@ -406,6 +419,7 @@ def get_model_from_name(model_name: str, device: str):
* virchow2
* hoptimus0
* hoptimus0_spectra
* mettle15
* h0mini
* hoptimus1
* genbio-pathfm
Expand Down Expand Up @@ -819,3 +833,31 @@ def get_genbio_pathfm(ckpt_path: str):

return model, transform


def get_mettle(ckpt_path: str):
"""
Adapted from:
- https://huggingface.co/slideflow-labs/mettle-1.5

:param ckpt_path: path to the folder with the stored checkpoint and model code.
"""
from torchvision import transforms

# Model
model = AutoModel.from_pretrained(ckpt_path, trust_remote_code=True)

# Transform (as specified in the model's preprocessor_config.json)
transform = transforms.Compose(
[
transforms.Resize(
(224, 224), interpolation=transforms.InterpolationMode.BICUBIC
),
transforms.ToTensor(),
transforms.Normalize(
mean=(0.707223, 0.578729, 0.703617),
std=(0.211883, 0.230117, 0.177517),
),
]
)

return model, transform
1 change: 1 addition & 0 deletions src/thunder/utils/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ class ModelConstants(Enum):
"hiboul",
"hoptimus0",
"hoptimus0_spectra",
"mettle15",
"h0mini",
"hoptimus1",
"genbio-pathfm",
Expand Down
Loading