From a743d90ed6cb2de888bec0b438d2da7a532a1496 Mon Sep 17 00:00:00 2001 From: Florence Bockting Date: Mon, 2 Nov 2026 12:52:00 +0200 Subject: [PATCH 1/4] feat: signal each class of posterior warning once in loops --- NEWS.md | 2 ++ R/E_loo.R | 20 +++++++++++--------- R/effective_sample_sizes.R | 5 +++-- R/helpers.R | 20 ++++++++++++++++++++ R/loo_moment_matching.R | 4 +++- tests/testthat/test_0_helpers.R | 26 ++++++++++++++++++++++++++ 6 files changed, 65 insertions(+), 12 deletions(-) diff --git a/NEWS.md b/NEWS.md index bcc1e16c..102de77c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -11,6 +11,8 @@ comparison-model minus reference-model by @florence-bockting in #391 * Fix `crps()` and `scrps()` ignoring the `permutations` argument for numeric input by @thanasibakis in #398 * Speed up the Bayesian bootstrap in `pseudobma_weights()` by @VisruthSK in #368 +* Signal each class of posterior warning once per call in `relative_eff()`, +`E_loo()`, and `loo_moment_match()` by @florence-bockting in #XXX # loo 2.10.1 diff --git a/R/E_loo.R b/R/E_loo.R index e360c3ef..3f783205 100644 --- a/R/E_loo.R +++ b/R/E_loo.R @@ -271,15 +271,17 @@ E_loo_khat.default <- function(x, psis_object, log_ratios, ...) { #' @export E_loo_khat.matrix <- function(x, psis_object, log_ratios, ...) { tail_lengths <- attr(psis_object, "tail_len") - if (is.null(x)) { - sapply(seq_len(ncol(log_ratios)), function(i) { - .E_loo_khat_i(x, log_ratios[, i], tail_lengths[i]) - }) - } else { - sapply(seq_len(ncol(log_ratios)), function(i) { - .E_loo_khat_i(x[, i], log_ratios[, i], tail_lengths[i]) - }) - } + .collect_warnings( + if (is.null(x)) { + sapply(seq_len(ncol(log_ratios)), function(i) { + .E_loo_khat_i(x, log_ratios[, i], tail_lengths[i]) + }) + } else { + sapply(seq_len(ncol(log_ratios)), function(i) { + .E_loo_khat_i(x[, i], log_ratios[, i], tail_lengths[i]) + }) + } + ) } #' Compute function-specific khat estimates diff --git a/R/effective_sample_sizes.R b/R/effective_sample_sizes.R index 360a3098..3aa25d7d 100644 --- a/R/effective_sample_sizes.R +++ b/R/effective_sample_sizes.R @@ -63,7 +63,7 @@ relative_eff.array <- function(x, ..., cores = getOption("mc.cores", 1)) { S <- prod(dim(x)[1:2]) # posterior sample size = iter * chains if (cores == 1) { - n_eff_vec <- apply(x, 3, posterior::ess_mean) + n_eff_vec <- .collect_warnings(apply(x, 3, posterior::ess_mean)) } else { if (!os_is_windows()) { n_eff_list <- @@ -105,7 +105,7 @@ relative_eff.function <- N <- dim(data)[1] if (cores == 1) { - n_eff_list <- + n_eff_list <- .collect_warnings( lapply( X = seq_len(N), FUN = function(i) { @@ -113,6 +113,7 @@ relative_eff.function <- relative_eff.default(as.vector(val_i), chain_id = chain_id, cores = 1) } ) + ) } else { if (!os_is_windows()) { n_eff_list <- diff --git a/R/helpers.R b/R/helpers.R index 37d407a8..d9d2ca8b 100644 --- a/R/helpers.R +++ b/R/helpers.R @@ -280,3 +280,23 @@ release_questions <- function() { is_constant <- function(x, tol = .Machine$double.eps) { abs(max(x) - min(x)) < tol } + +#' Collect posterior warnings and signal each class once +#' +#' @noRd +#' @param expr Expression to evaluate. +#' @return The value of `expr`. +.collect_warnings <- function(expr) { + collected <- list() + out <- withCallingHandlers( + expr, + posterior_warning = function(cnd) { + collected[[class(cnd)[1]]] <<- cnd + invokeRestart("muffleWarning") + } + ) + for (cnd in collected) { + warning(cnd) + } + out +} diff --git a/R/loo_moment_matching.R b/R/loo_moment_matching.R index 08d6ced6..4ac89103 100644 --- a/R/loo_moment_matching.R +++ b/R/loo_moment_matching.R @@ -123,7 +123,9 @@ loo_moment_match.default <- function(x, loo, post_draws, log_lik_i, } if (cores == 1) { - mm_list <- lapply(X = I, FUN = function(i) loo_moment_match_i_fun(i)) + mm_list <- .collect_warnings( + lapply(X = I, FUN = function(i) loo_moment_match_i_fun(i)) + ) } else { if (!os_is_windows()) { diff --git a/tests/testthat/test_0_helpers.R b/tests/testthat/test_0_helpers.R index a0add421..8628af1c 100644 --- a/tests/testthat/test_0_helpers.R +++ b/tests/testthat/test_0_helpers.R @@ -91,3 +91,29 @@ test_that("loo_cores works", { expect_warning(expect_equal(loo_cores(10), 2), "deprecated") options(loo.cores = NULL) }) + +test_that(".collect_warnings() signals each posterior class once", { + warn_classed <- function(class) { + warning(structure( + class = c(class, "posterior_warning", "warning", "condition"), + list(message = class, call = NULL) + )) + } + caught <- character() + out <- withCallingHandlers( + .collect_warnings({ + warn_classed("a") + warn_classed("a") + .collect_warnings(warn_classed("b")) + warn_classed("b") + warning("other") + 1 + }), + warning = function(cnd) { + caught <<- c(caught, conditionMessage(cnd)) + invokeRestart("muffleWarning") + } + ) + expect_equal(out, 1) + expect_equal(caught, c("other", "a", "b")) +}) From 45a7c819d632675290a6980fc6157d828916ad23 Mon Sep 17 00:00:00 2001 From: Florence Bockting Date: Mon, 2 Nov 2026 13:03:04 +0200 Subject: [PATCH 2/4] chore: update PR number in NEWS.md --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 102de77c..6040665c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -12,7 +12,7 @@ comparison-model minus reference-model by @florence-bockting in #391 input by @thanasibakis in #398 * Speed up the Bayesian bootstrap in `pseudobma_weights()` by @VisruthSK in #368 * Signal each class of posterior warning once per call in `relative_eff()`, -`E_loo()`, and `loo_moment_match()` by @florence-bockting in #XXX +`E_loo()`, and `loo_moment_match()` by @florence-bockting in #416 # loo 2.10.1 From 9d342decee7b49baeee2b752ba9c9b484376627a Mon Sep 17 00:00:00 2001 From: Florence Bockting Date: Mon, 9 Nov 2026 10:37:58 +0200 Subject: [PATCH 3/4] fix: address review of posterior warning collector --- DESCRIPTION | 2 +- NEWS.md | 2 +- R/helpers.R | 21 ++++++++++++----- tests/testthat/test_0_helpers.R | 40 ++++++++++++++++++++++++++++++++- 4 files changed, 56 insertions(+), 9 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index d98e2a39..9c631533 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -51,7 +51,7 @@ Suggests: rstanarm (>= 2.19.0), rstantools, spdep, - testthat (>= 3.0) + testthat (>= 3.1.7) VignetteBuilder: knitr Config/testthat/edition: 3 diff --git a/NEWS.md b/NEWS.md index 6040665c..59998c3a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -12,7 +12,7 @@ comparison-model minus reference-model by @florence-bockting in #391 input by @thanasibakis in #398 * Speed up the Bayesian bootstrap in `pseudobma_weights()` by @VisruthSK in #368 * Signal each class of posterior warning once per call in `relative_eff()`, -`E_loo()`, and `loo_moment_match()` by @florence-bockting in #416 +`E_loo()`, and `loo_moment_match()` (requires posterior >= 1.7.1) by @florence-bockting in #416 # loo 2.10.1 diff --git a/R/helpers.R b/R/helpers.R index d9d2ca8b..7d56b939 100644 --- a/R/helpers.R +++ b/R/helpers.R @@ -288,15 +288,24 @@ is_constant <- function(x, tol = .Machine$double.eps) { #' @return The value of `expr`. .collect_warnings <- function(expr) { collected <- list() - out <- withCallingHandlers( + counts <- list() + on.exit(for (cls in names(collected)) { + cnd <- collected[[cls]] + if (counts[[cls]] > 1) { + cnd$message <- paste0(cnd$message, " (", counts[[cls]], " times)") + } + warning(cnd) + }) + withCallingHandlers( expr, posterior_warning = function(cnd) { - collected[[class(cnd)[1]]] <<- cnd + cls <- class(cnd)[1] + if (is.null(collected[[cls]])) { + collected[[cls]] <<- cnd + counts[[cls]] <<- 0 + } + counts[[cls]] <<- counts[[cls]] + 1 invokeRestart("muffleWarning") } ) - for (cnd in collected) { - warning(cnd) - } - out } diff --git a/tests/testthat/test_0_helpers.R b/tests/testthat/test_0_helpers.R index 8628af1c..f5b510b3 100644 --- a/tests/testthat/test_0_helpers.R +++ b/tests/testthat/test_0_helpers.R @@ -115,5 +115,43 @@ test_that(".collect_warnings() signals each posterior class once", { } ) expect_equal(out, 1) - expect_equal(caught, c("other", "a", "b")) + expect_equal(caught, c("other", "a (2 times)", "b (2 times)")) +}) + +test_that(".collect_warnings() keeps the first warning and signals it after an error", { + warn_a <- function(msg) { + warning(structure( + class = c("a", "posterior_warning", "warning", "condition"), + list(message = msg, call = NULL) + )) + } + w <- capture_warnings(try(.collect_warnings({ + warn_a("first") + warn_a("second") + stop("boom") + }), silent = TRUE)) + expect_equal(w, "first (2 times)") +}) + +test_that("relative_eff() signals each posterior warning class once", { + local_mocked_bindings( + ess_mean = function(x, ...) { + warning(structure( + class = c("posterior_warning_x", "posterior_warning", "warning", "condition"), + list(message = "mocked", call = NULL) + )) + 1 + }, + .package = "posterior" + ) + w <- capture_warnings(relative_eff(exp(example_loglik_array()), cores = 1)) + expect_equal(w, "mocked (32 times)") +}) + +test_that("relative_eff() signals the posterior ESS warning once", { + skip_if_not(packageVersion("posterior") >= "1.7.1") + set.seed(1) + x <- array(rep(c(1, 2), 300) + rnorm(600, 0, 0.1), dim = c(100, 2, 3)) + w <- capture_warnings(relative_eff(x, cores = 1)) + expect_equal(w, "The ESS has been capped to avoid unstable estimates. (3 times)") }) From a70c12460d9d62428a8c69282b3b6b9c2f68fcc7 Mon Sep 17 00:00:00 2001 From: Florence Bockting Date: Tue, 10 Nov 2026 11:24:26 +0200 Subject: [PATCH 4/4] fix: minor typo --- tests/testthat/test_0_helpers.R | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/testthat/test_0_helpers.R b/tests/testthat/test_0_helpers.R index d08d73a0..60e5c28d 100644 --- a/tests/testthat/test_0_helpers.R +++ b/tests/testthat/test_0_helpers.R @@ -154,6 +154,8 @@ test_that("relative_eff() signals the posterior ESS warning once", { x <- array(rep(c(1, 2), 300) + rnorm(600, 0, 0.1), dim = c(100, 2, 3)) w <- capture_warnings(relative_eff(x, cores = 1)) expect_equal(w, "The ESS has been capped to avoid unstable estimates. (3 times)") +}) + test_that("is_constant uses a tolerance relative to the magnitude of x", { expect_true(is_constant(rep(0, 5))) expect_true(is_constant(rep(1e-300, 5)))