From b971d698b387ec5faa40e55fc72f72a0996735c7 Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Tue, 29 Sep 2026 08:55:53 +0200 Subject: [PATCH 1/5] run_benchmark: write the commit and launch time into task_info.yaml --- CHANGELOG.md | 2 ++ src/workflows/run_benchmark/main.nf | 12 +++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96292324..4e92166e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ * `file_train_mod1`, `file_train_mod2`, `file_test_mod1`, `file_test_mod2`: Declare `uns["modality"]`, which `process_dataset` already writes and which six methods and the `run_benchmark` workflow already read (PR #29). +* `run_benchmark`: Write the commit the workflow ran from and the launch time into `task_info.yaml`, instead of publishing `_viash.yaml` verbatim (ported from openproblems-bio/task_template#18). + * `mse`: Write the unbounded maximum as `"+.inf"` rather than `"+inf"`, which is the literal the metric schema accepts (PR #31). * `cellmapper_linear`: Write the unmasked variants as `mask_var: null` rather than `mask_var: None`, which YAML reads as the string `"None"` and which would resolve to `adata.var["None"]` (PR #38). diff --git a/src/workflows/run_benchmark/main.nf b/src/workflows/run_benchmark/main.nf index 222a313c..dadb1303 100644 --- a/src/workflows/run_benchmark/main.nf +++ b/src/workflows/run_benchmark/main.nf @@ -196,7 +196,17 @@ workflow run_wf { def metric_configs_file = tempFile("metric_configs.yaml") metric_configs_file.write(metric_configs_yaml_blob) - def task_info_file = meta.resources_dir.resolve("_viash.yaml") + // store the task info in a file + def task_info = readYaml(meta.resources_dir.resolve("_viash.yaml")) + // commitId is null when nextflow runs from a local checkout instead of a revision + if (workflow.commitId) { + task_info.commit = workflow.commitId + } + // the launch time -- workflow.complete is only known once the run is over + task_info.timestamp = workflow.start.toInstant() + .truncatedTo(java.time.temporal.ChronoUnit.SECONDS).toString() + def task_info_file = tempFile("task_info.yaml") + task_info_file.write(toYamlBlob(task_info)) // create output def new_state = [ From 23cbc49a894ec33389917bf8cf662fecbc993511 Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Tue, 29 Sep 2026 08:56:04 +0200 Subject: [PATCH 2/5] Update common submodule --- common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common b/common index 6eda5dd0..7e2d6bab 160000 --- a/common +++ b/common @@ -1 +1 @@ -Subproject commit 6eda5dd0b9c46b43ee36fc379152e079c26f25d3 +Subproject commit 7e2d6bab55d98afa808f39b339f7828551c90cd5 From c1b3d55f3b312a3fcc2dc57622d6dd956c81f515 Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Tue, 29 Sep 2026 08:56:37 +0200 Subject: [PATCH 3/5] run_benchmark: replace --method_ids with include/exclude filters --- CHANGELOG.md | 2 ++ src/workflows/run_benchmark/config.vsh.yaml | 37 +++++++++++++++++++-- src/workflows/run_benchmark/main.nf | 22 +++++++++++- 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e92166e..2c6b2c56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ * `run_benchmark`: Write the commit the workflow ran from and the launch time into `task_info.yaml`, instead of publishing `_viash.yaml` verbatim (ported from openproblems-bio/task_template#18). +* `run_benchmark`: Replace `--method_ids` with `--methods_include`/`--methods_exclude`, and add `--metrics_include`/`--metrics_exclude` (ported from openproblems-bio/task_template#20). + * `mse`: Write the unbounded maximum as `"+.inf"` rather than `"+inf"`, which is the literal the metric schema accepts (PR #31). * `cellmapper_linear`: Write the unmasked variants as `mask_var: null` rather than `mask_var: None`, which YAML reads as the string `"None"` and which would resolve to `adata.var["None"]` (PR #38). diff --git a/src/workflows/run_benchmark/config.vsh.yaml b/src/workflows/run_benchmark/config.vsh.yaml index c9221032..65d7ae71 100644 --- a/src/workflows/run_benchmark/config.vsh.yaml +++ b/src/workflows/run_benchmark/config.vsh.yaml @@ -47,18 +47,49 @@ argument_groups: required: true direction: output default: task_info.yaml - - name: Methods + - name: Method filtering + description: | + Use these arguments to filter methods by name. By default, all methods are + run. If `--methods_include` is defined, only those methods are run. If + `--methods_exclude` is defined, all methods except those specified are run. + These arguments are mutually exclusive, so only `--methods_include` OR + `--methods_exclude` can set but not both. arguments: - - name: "--method_ids" + - name: "--methods_include" type: string multiple: true - description: A list of method ids to run. If not specified, all methods will be run. + description: | + A list of method ids to include. If specified, only these methods will be run. + - name: "--methods_exclude" + type: string + multiple: true + description: | + A list of method ids to exclude. If specified, all methods except the ones listed will be run. + - name: Metric filtering + description: | + Use these arguments to filter metrics by name. By default, all metrics are + run. If `--metrics_include` is defined, only those metrics are run. If + `--metrics_exclude` is defined, all metrics except those specified are run. + These arguments are mutually exclusive, so only `--metrics_include` OR + `--metrics_exclude` can set but not both. + arguments: + - name: "--metrics_include" + type: string + multiple: true + description: | + A list of metric ids to include. If specified, only these metrics will be run. + - name: "--metrics_exclude" + type: string + multiple: true + description: | + A list of metric ids to exclude. If specified, all metrics except the ones listed will be run. resources: - type: nextflow_script path: main.nf entrypoint: run_wf - type: file path: /_viash.yaml + - path: /common/nextflow_helpers/helper.nf dependencies: - name: utils/extract_uns_metadata repository: openproblems diff --git a/src/workflows/run_benchmark/main.nf b/src/workflows/run_benchmark/main.nf index dadb1303..42ee5b91 100644 --- a/src/workflows/run_benchmark/main.nf +++ b/src/workflows/run_benchmark/main.nf @@ -1,3 +1,5 @@ +include { checkItemAllowed } from "${meta.resources_dir}/helper.nf" + workflow auto { findStates(params, meta.config) | meta.workflow.run( @@ -90,7 +92,13 @@ workflow run_wf { // if the preferred normalisation is none at all, // we can pass whichever dataset we want def norm_check = (norm == "log_cp10k" && pref == "counts") || norm == pref - def method_check = !state.method_ids || state.method_ids.contains(comp.config.name) + def method_check = checkItemAllowed( + comp.config.name, + state.methods_include, + state.methods_exclude, + "methods_include", + "methods_exclude" + ) method_check && norm_check }, @@ -125,6 +133,18 @@ workflow run_wf { // run all metrics | runEach( components: metrics, + + // use the 'filter' argument to only run the metrics the user asked for + filter: { id, state, comp -> + checkItemAllowed( + comp.config.name, + state.metrics_include, + state.metrics_exclude, + "metrics_include", + "metrics_exclude" + ) + }, + id: { id, state, comp -> id + "." + comp.config.name }, From 75f6aee5bd209f4e972fd79d21a9e529e9c377b1 Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Tue, 29 Sep 2026 09:59:41 +0200 Subject: [PATCH 4/5] run_benchmark: run parameterised methods once per paramset --- CHANGELOG.md | 2 + src/workflows/run_benchmark/config.vsh.yaml | 22 +++++++++++ src/workflows/run_benchmark/main.nf | 43 ++++++++++++++------- 3 files changed, 54 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c6b2c56..ecc8528a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ * `run_benchmark`: Replace `--method_ids` with `--methods_include`/`--methods_exclude`, and add `--metrics_include`/`--metrics_exclude` (ported from openproblems-bio/task_template#20). +* `run_benchmark`: Run parameterised methods once per named paramset from `info.variants` or the new `--paramsets` file, tag scores with `paramset_name`/`paramset`, and allow `--methods_include`/`--methods_exclude` to target `.`. The existing `info.variants` of `cellmapper_linear` and `cellmapper_scvi` are now run by default (ported from openproblems-bio/task_template#23). + * `mse`: Write the unbounded maximum as `"+.inf"` rather than `"+inf"`, which is the literal the metric schema accepts (PR #31). * `cellmapper_linear`: Write the unmasked variants as `mask_var: null` rather than `mask_var: None`, which YAML reads as the string `"None"` and which would resolve to `adata.var["None"]` (PR #38). diff --git a/src/workflows/run_benchmark/config.vsh.yaml b/src/workflows/run_benchmark/config.vsh.yaml index 65d7ae71..9ee417f2 100644 --- a/src/workflows/run_benchmark/config.vsh.yaml +++ b/src/workflows/run_benchmark/config.vsh.yaml @@ -55,16 +55,38 @@ argument_groups: These arguments are mutually exclusive, so only `--methods_include` OR `--methods_exclude` can set but not both. arguments: + - name: "--paramsets" + type: file + direction: input + required: false + description: | + A yaml file mapping method ids to named parameter sets, e.g.: + + cellmapper_linear: + pca_hvg: + fallback_representation: joint_pca + mask_var: hvg + cca_hvg: + fallback_representation: fast_cca + mask_var: hvg + + Each listed method is run once per parameter set, with the given arguments + and with `paramset_name`/`paramset` attached to its scores. + If not provided, the parameter sets default to the `info.variants` + defined in the method components' configs. A method without + `info.variants` is run once with its default parameter set. - name: "--methods_include" type: string multiple: true description: | A list of method ids to include. If specified, only these methods will be run. + For parameterised methods, a single paramset can be targeted as `.`, e.g. `cellmapper_linear.cellmapper-pca`. - name: "--methods_exclude" type: string multiple: true description: | A list of method ids to exclude. If specified, all methods except the ones listed will be run. + For parameterised methods, a single paramset can be excluded as `.`, e.g. `cellmapper_linear.cellmapper-pca`. - name: Metric filtering description: | Use these arguments to filter metrics by name. By default, all metrics are diff --git a/src/workflows/run_benchmark/main.nf b/src/workflows/run_benchmark/main.nf index 42ee5b91..e6ce777a 100644 --- a/src/workflows/run_benchmark/main.nf +++ b/src/workflows/run_benchmark/main.nf @@ -1,4 +1,4 @@ -include { checkItemAllowed } from "${meta.resources_dir}/helper.nf" +include { checkItemAllowed; paramsetsFromVariants; expandParamsets; methodMatchesParamset; checkMethodAllowed } from "${meta.resources_dir}/helper.nf" workflow auto { findStates(params, meta.config) @@ -81,34 +81,47 @@ workflow run_wf { ***************************/ score_ch = dataset_ch + // expand the channel so parameterised methods run once per paramset. + // the paramsets are read from the --paramsets file if provided, and + // default to the method components' info.variants. + | flatMap { id, state -> + def method_paramsets = state.paramsets + ? readYaml(state.paramsets) + : paramsetsFromVariants(methods) + expandParamsets(id, state, method_paramsets) + } + // run all methods | runEach( components: methods, - // use the 'filter' argument to only run a method on the normalisation the component is asking for + // use the 'filter' argument to only run a method on the normalisation the + // component is asking for, to match paramset-tagged states to their method, + // and to filter by --methods_include/--methods_exclude filter: { id, state, comp -> def norm = state.rna_norm def pref = comp.config.info.preferred_normalization // if the preferred normalisation is none at all, // we can pass whichever dataset we want def norm_check = (norm == "log_cp10k" && pref == "counts") || norm == pref - def method_check = checkItemAllowed( + def paramset_check = methodMatchesParamset(state, comp.config.name) + def method_check = checkMethodAllowed( comp.config.name, + state.paramset_name, state.methods_include, - state.methods_exclude, - "methods_include", - "methods_exclude" + state.methods_exclude ) - method_check && norm_check + norm_check && paramset_check && method_check }, - // define a new 'id' by appending the method name to the dataset id + // define a new 'id' by appending the method name and paramset name to the dataset id id: { id, state, comp -> - id + "." + comp.config.name + id + "." + comp.config.name + (state.paramset_name ? "." + state.paramset_name : "") }, - // use 'fromState' to fetch the arguments the component requires from the overall state + // use 'fromState' to fetch the arguments the component requires from the overall state, + // along with the paramset arguments (if any) fromState: { id, state, comp -> def new_args = [ input_train_mod1: state.input_train_mod1, @@ -118,7 +131,7 @@ workflow run_wf { if (comp.config.info.type == "control_method") { new_args.input_test_mod2 = state.input_test_mod2 } - new_args + new_args + (state.paramset ?: [:]) }, // use 'toState' to publish that component's outputs to the overall state @@ -161,13 +174,17 @@ workflow run_wf { ] } ) - // extract the scores + // extract the scores, tagged with the paramset used for the method + // (null for control methods and methods without paramsets) | extract_uns_metadata.run( key: "extract_scores", fromState: [input: "metric_output"], toState: { id, output, state -> + def uns = readYaml(output.output).uns + uns.paramset_name = state.paramset_name + uns.paramset = state.paramset state + [ - score_uns: readYaml(output.output).uns + score_uns: uns ] } ) From 3fbe74e74cfdea322c6039db197181efb461f6af Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Tue, 29 Sep 2026 10:15:46 +0200 Subject: [PATCH 5/5] cellmapper_linear: use "none" instead of null for unmasked variants --- CHANGELOG.md | 2 ++ src/methods/cellmapper_linear/config.vsh.yaml | 6 +++--- src/methods/cellmapper_linear/script.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ecc8528a..1cf2a082 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,8 @@ ## BUG FIXES +* `cellmapper_linear`: Write the unmasked variants as `mask_var: "none"` rather than `mask_var: null`. Viash drops null values from the config, so these variants fell back to the default `mask_var: "hvg"` and duplicated their `-hvg` counterparts. + * `guanlab_dengkw_pm`: Restore the consensus scheme of the original submission -- five reshuffles of the batches into two halves, ten kernel ridge models averaged. The port had replaced it with a single fixed two-way split for ADT pairs and leave-one-batch-out otherwise, so the result depended on the order the batches happened to come in. `--n_repeats` and `--seed` are now arguments; the unused `--distance_method` and `--n_pcs` are gone (PR #32). * `guanlab_dengkw_pm`: Only map the predictions back through the mod2 SVD when that SVD was actually fitted. When the target had fewer features than `n_mod2`, the method raised `NameError: embedder_mod2`. Unsupported modality pairs now exit 99 (non-applicable) instead of raising a `KeyError` (PR #32). diff --git a/src/methods/cellmapper_linear/config.vsh.yaml b/src/methods/cellmapper_linear/config.vsh.yaml index 94cae038..33e03ddf 100644 --- a/src/methods/cellmapper_linear/config.vsh.yaml +++ b/src/methods/cellmapper_linear/config.vsh.yaml @@ -16,7 +16,7 @@ info: variants: cellmapper-pca: fallback_representation: joint_pca - mask_var: null + mask_var: "none" kernel_method: hnoca cellmapper-pca-hvg: fallback_representation: joint_pca @@ -28,7 +28,7 @@ info: kernel_method: gauss cellmapper-cca: fallback_representation: fast_cca - mask_var: null + mask_var: "none" kernel_method: hnoca cellmapper-cca-hvg: fallback_representation: fast_cca @@ -47,7 +47,7 @@ arguments: - name: "--mask_var" type: "string" default: "hvg" - description: Variable to mask for fallback representation. + description: Variable to mask for fallback representation, or "none" to use all features. - name: "--kernel_method" type: "string" choices: ["hnoca", "gauss"] diff --git a/src/methods/cellmapper_linear/script.py b/src/methods/cellmapper_linear/script.py index 194235e2..7059f7fa 100644 --- a/src/methods/cellmapper_linear/script.py +++ b/src/methods/cellmapper_linear/script.py @@ -43,7 +43,7 @@ knn_method="sklearn" if n_obs < 60000 else "pynndescent", fallback_representation=par['fallback_representation'], n_neighbors=par['n_neighbors'], - fallback_kwargs={"mask_var": par['mask_var']}, + fallback_kwargs={"mask_var": None if par['mask_var'] == "none" else par['mask_var']}, ) cmap.compute_mapping_matrix(kernel_method=par['kernel_method'])