diff --git a/README.md b/README.md index 27b95e3..8a2ac4f 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ An API and command line interface (CLI) are provided to allow users to download * Virchow2: https://huggingface.co/paige-ai/Virchow2 * H-optimus-0: https://huggingface.co/bioptimus/H-optimus-0 * H-optimus-1: https://huggingface.co/bioptimus/H-optimus-1 +* OpenMidnight-1.1-SPECTRA: https://huggingface.co/medarc/OpenMidnight-1.1-SPECTRA * H-optimus-0-SPECTRA: https://huggingface.co/medarc/H-optimus-0-SPECTRA * Virchow-SPECTRA: https://huggingface.co/medarc/Virchow-SPECTRA * Midnight-12k-SPECTRA: https://huggingface.co/medarc/Midnight-12k-SPECTRA diff --git a/src/thunder/benchmark.py b/src/thunder/benchmark.py index 13a1496..b433552 100644 --- a/src/thunder/benchmark.py +++ b/src/thunder/benchmark.py @@ -21,7 +21,7 @@ def benchmark( where options are: - dataset: *bach*, *bracs*, *break_his*, *ccrcc*, *crc*, *esca*, *mhist*, *ocelot*, *pannuke*, *patch_camelyon*, *segpath_epithelial*, *segpath_lymphocytes*, *tcga_crc_msi*, *tcga_tils*, *tcga_uniform*, *wilds* - - model: *hiboub*, *hiboul*, *hoptimus0*, *hoptimus1*, *genbio-pathfm*, *hoptimus0_spectra*, *midnight12k_spectra*, *virchow_spectra*, *midnight*, *phikon*, *phikon2*, *uni*, *uni2h*, *virchow*, *virchow2*, *conch*, *titan*, *keep*, *musk*, *plip*, *quiltnetb32*, *dinov2base*, *dinov2large*, *vitbasepatch16224in21k*, *vitlargepatch16224in21k*, *clipvitbasepatch32*, *clipvitlargepatch14* + - model: *hiboub*, *hiboul*, *hoptimus0*, *hoptimus1*, *genbio-pathfm*, *openmidnight11_spectra*, *hoptimus0_spectra*, *midnight12k_spectra*, *virchow_spectra*, *midnight*, *phikon*, *phikon2*, *uni*, *uni2h*, *virchow*, *virchow2*, *conch*, *titan*, *keep*, *musk*, *plip*, *quiltnetb32*, *dinov2base*, *dinov2large*, *vitbasepatch16224in21k*, *vitlargepatch16224in21k*, *clipvitbasepatch32*, *clipvitlargepatch14* - task: *adversarial_attack*, *alignment_scoring*, *image_retrieval*, *knn*, *linear_probing*, *pre_computing_embeddings*, *segmentation*, *simple_shot*, *transformation_invariance*, *zero_shot_vlm* - loading_mode: *online_loading*, *image_pre_loading*, *embedding_pre_loading* diff --git a/src/thunder/config/pretrained_model/openmidnight11_spectra.yaml b/src/thunder/config/pretrained_model/openmidnight11_spectra.yaml new file mode 100644 index 0000000..8aa9af2 --- /dev/null +++ b/src/thunder/config/pretrained_model/openmidnight11_spectra.yaml @@ -0,0 +1,8 @@ +model_name: openmidnight11_spectra +type: timm +vlm: false +emb_dim: 1536 +ckpt_path: ${oc.env:THUNDER_BASE_DATA_FOLDER}/pretrained_ckpts/openmidnight1.1spectra/model.safetensors +hf_tag: hf-hub:medarc/OpenMidnight-1.1-SPECTRA + +timm_kwargs: {} diff --git a/src/thunder/models/download.py b/src/thunder/models/download.py index 22cb32d..8a09a21 100644 --- a/src/thunder/models/download.py +++ b/src/thunder/models/download.py @@ -102,6 +102,10 @@ "kaiko-ai/midnight", "model.safetensors", ), # Midnight-12k (https://huggingface.co/kaiko-ai/midnight) + "openmidnight11_spectra": ( + "medarc/OpenMidnight-1.1-SPECTRA", + "model.safetensors", + ), # OpenMidnight-1.1-SPECTRA (https://huggingface.co/medarc/OpenMidnight-1.1-SPECTRA) "openmidnight": ( "SophontAI/OpenMidnight", "teacher_checkpoint_load.pt", @@ -185,6 +189,7 @@ def download_models(models: Union[List[str], str]) -> None: * midnight * midnight12k_spectra * openmidnight + * openmidnight11_spectra * kaiko_vits8 * kaiko_vits16 * kaiko_vitb8 @@ -277,7 +282,13 @@ def download_model(model: str) -> None: ] if local_dir_tag != "plip": extra_files.extend(["merges.txt", "vocab.json"]) - elif local_dir_tag in ["midnight", "hoptimus0spectra", "midnight12kspectra", "virchowspectra"]: + elif local_dir_tag in [ + "midnight", + "openmidnight1.1spectra", + "hoptimus0spectra", + "midnight12kspectra", + "virchowspectra" + ]: extra_files = ["config.json"] elif local_dir_tag == "titan": extra_files = [ diff --git a/src/thunder/models/pretrained_models.py b/src/thunder/models/pretrained_models.py index 9c8324b..518505f 100644 --- a/src/thunder/models/pretrained_models.py +++ b/src/thunder/models/pretrained_models.py @@ -380,6 +380,7 @@ def extract_embedding(src, pretrained_model, task_type="linear_probing"): elif model_cfg.model_name in [ "hoptimus0", "hoptimus1", + "openmidnight11_spectra", "hoptimus0_spectra", ]: nb_reg = 4 @@ -417,6 +418,7 @@ def get_model_from_name(model_name: str, device: str): * midnight * midnight12k_spectra * openmidnight + * openmidnight11_spectra * kaiko_vits8 * kaiko_vits16 * kaiko_vitb8 diff --git a/src/thunder/utils/constants.py b/src/thunder/utils/constants.py index da84387..9a61837 100644 --- a/src/thunder/utils/constants.py +++ b/src/thunder/utils/constants.py @@ -18,6 +18,7 @@ class ModelConstants(Enum): "midnight", "midnight12k_spectra", "openmidnight", + "openmidnight11_spectra", "phikon", "phikon2", "uni",