diff --git a/DESCRIPTION b/DESCRIPTION index 36e3b88c..daa0db08 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -51,7 +51,7 @@ Suggests: rstanarm (>= 2.19.0), rstantools, spdep, - testthat (>= 3.0) + testthat (>= 3.1.7) VignetteBuilder: knitr Config/testthat/edition: 3 diff --git a/NEWS.md b/NEWS.md index 1ed6bd3a..9f6f97cb 100644 --- a/NEWS.md +++ b/NEWS.md @@ -11,6 +11,8 @@ comparison-model minus reference-model by @florence-bockting in #391 * Fix `crps()` and `scrps()` ignoring the `permutations` argument for numeric input by @thanasibakis in #398 * Speed up the Bayesian bootstrap in `pseudobma_weights()` by @VisruthSK in #368 +* Signal each class of posterior warning once per call in `relative_eff()`, +`E_loo()`, and `loo_moment_match()` (requires posterior >= 1.7.1) by @florence-bockting in #416 * Use a tolerance relative to the magnitude of `x` in `is_constant()` by @avehtari in #407 diff --git a/R/E_loo.R b/R/E_loo.R index e360c3ef..3f783205 100644 --- a/R/E_loo.R +++ b/R/E_loo.R @@ -271,15 +271,17 @@ E_loo_khat.default <- function(x, psis_object, log_ratios, ...) { #' @export E_loo_khat.matrix <- function(x, psis_object, log_ratios, ...) { tail_lengths <- attr(psis_object, "tail_len") - if (is.null(x)) { - sapply(seq_len(ncol(log_ratios)), function(i) { - .E_loo_khat_i(x, log_ratios[, i], tail_lengths[i]) - }) - } else { - sapply(seq_len(ncol(log_ratios)), function(i) { - .E_loo_khat_i(x[, i], log_ratios[, i], tail_lengths[i]) - }) - } + .collect_warnings( + if (is.null(x)) { + sapply(seq_len(ncol(log_ratios)), function(i) { + .E_loo_khat_i(x, log_ratios[, i], tail_lengths[i]) + }) + } else { + sapply(seq_len(ncol(log_ratios)), function(i) { + .E_loo_khat_i(x[, i], log_ratios[, i], tail_lengths[i]) + }) + } + ) } #' Compute function-specific khat estimates diff --git a/R/effective_sample_sizes.R b/R/effective_sample_sizes.R index 360a3098..3aa25d7d 100644 --- a/R/effective_sample_sizes.R +++ b/R/effective_sample_sizes.R @@ -63,7 +63,7 @@ relative_eff.array <- function(x, ..., cores = getOption("mc.cores", 1)) { S <- prod(dim(x)[1:2]) # posterior sample size = iter * chains if (cores == 1) { - n_eff_vec <- apply(x, 3, posterior::ess_mean) + n_eff_vec <- .collect_warnings(apply(x, 3, posterior::ess_mean)) } else { if (!os_is_windows()) { n_eff_list <- @@ -105,7 +105,7 @@ relative_eff.function <- N <- dim(data)[1] if (cores == 1) { - n_eff_list <- + n_eff_list <- .collect_warnings( lapply( X = seq_len(N), FUN = function(i) { @@ -113,6 +113,7 @@ relative_eff.function <- relative_eff.default(as.vector(val_i), chain_id = chain_id, cores = 1) } ) + ) } else { if (!os_is_windows()) { n_eff_list <- diff --git a/R/helpers.R b/R/helpers.R index 962e93bc..3a653f60 100644 --- a/R/helpers.R +++ b/R/helpers.R @@ -285,3 +285,32 @@ is_constant <- function(x, tol = .Machine$double.eps) { # the equality check covers all-zero and all-infinite x r[1] == r[2] || (is.finite(spread) && spread <= tol * max(abs(r))) } + +#' Collect posterior warnings and signal each class once +#' +#' @noRd +#' @param expr Expression to evaluate. +#' @return The value of `expr`. +.collect_warnings <- function(expr) { + collected <- list() + counts <- list() + on.exit(for (cls in names(collected)) { + cnd <- collected[[cls]] + if (counts[[cls]] > 1) { + cnd$message <- paste0(cnd$message, " (", counts[[cls]], " times)") + } + warning(cnd) + }) + withCallingHandlers( + expr, + posterior_warning = function(cnd) { + cls <- class(cnd)[1] + if (is.null(collected[[cls]])) { + collected[[cls]] <<- cnd + counts[[cls]] <<- 0 + } + counts[[cls]] <<- counts[[cls]] + 1 + invokeRestart("muffleWarning") + } + ) +} diff --git a/R/loo_moment_matching.R b/R/loo_moment_matching.R index 08d6ced6..4ac89103 100644 --- a/R/loo_moment_matching.R +++ b/R/loo_moment_matching.R @@ -123,7 +123,9 @@ loo_moment_match.default <- function(x, loo, post_draws, log_lik_i, } if (cores == 1) { - mm_list <- lapply(X = I, FUN = function(i) loo_moment_match_i_fun(i)) + mm_list <- .collect_warnings( + lapply(X = I, FUN = function(i) loo_moment_match_i_fun(i)) + ) } else { if (!os_is_windows()) { diff --git a/tests/testthat/test_0_helpers.R b/tests/testthat/test_0_helpers.R index 01a808d6..60e5c28d 100644 --- a/tests/testthat/test_0_helpers.R +++ b/tests/testthat/test_0_helpers.R @@ -92,6 +92,70 @@ test_that("loo_cores works", { options(loo.cores = NULL) }) +test_that(".collect_warnings() signals each posterior class once", { + warn_classed <- function(class) { + warning(structure( + class = c(class, "posterior_warning", "warning", "condition"), + list(message = class, call = NULL) + )) + } + caught <- character() + out <- withCallingHandlers( + .collect_warnings({ + warn_classed("a") + warn_classed("a") + .collect_warnings(warn_classed("b")) + warn_classed("b") + warning("other") + 1 + }), + warning = function(cnd) { + caught <<- c(caught, conditionMessage(cnd)) + invokeRestart("muffleWarning") + } + ) + expect_equal(out, 1) + expect_equal(caught, c("other", "a (2 times)", "b (2 times)")) +}) + +test_that(".collect_warnings() keeps the first warning and signals it after an error", { + warn_a <- function(msg) { + warning(structure( + class = c("a", "posterior_warning", "warning", "condition"), + list(message = msg, call = NULL) + )) + } + w <- capture_warnings(try(.collect_warnings({ + warn_a("first") + warn_a("second") + stop("boom") + }), silent = TRUE)) + expect_equal(w, "first (2 times)") +}) + +test_that("relative_eff() signals each posterior warning class once", { + local_mocked_bindings( + ess_mean = function(x, ...) { + warning(structure( + class = c("posterior_warning_x", "posterior_warning", "warning", "condition"), + list(message = "mocked", call = NULL) + )) + 1 + }, + .package = "posterior" + ) + w <- capture_warnings(relative_eff(exp(example_loglik_array()), cores = 1)) + expect_equal(w, "mocked (32 times)") +}) + +test_that("relative_eff() signals the posterior ESS warning once", { + skip_if_not(packageVersion("posterior") >= "1.7.1") + set.seed(1) + x <- array(rep(c(1, 2), 300) + rnorm(600, 0, 0.1), dim = c(100, 2, 3)) + w <- capture_warnings(relative_eff(x, cores = 1)) + expect_equal(w, "The ESS has been capped to avoid unstable estimates. (3 times)") +}) + test_that("is_constant uses a tolerance relative to the magnitude of x", { expect_true(is_constant(rep(0, 5))) expect_true(is_constant(rep(1e-300, 5)))