Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion DESCRIPTION
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ Suggests:
rstanarm (>= 2.19.0),
rstantools,
spdep,
testthat (>= 3.0)
testthat (>= 3.1.7)
VignetteBuilder:
knitr
Config/testthat/edition: 3
Expand Down
2 changes: 2 additions & 0 deletions NEWS.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ comparison-model minus reference-model by @florence-bockting in #391
* Fix `crps()` and `scrps()` ignoring the `permutations` argument for numeric
input by @thanasibakis in #398
* Speed up the Bayesian bootstrap in `pseudobma_weights()` by @VisruthSK in #368
* Signal each class of posterior warning once per call in `relative_eff()`,
`E_loo()`, and `loo_moment_match()` (requires posterior >= 1.7.1) by @florence-bockting in #416
* Use a tolerance relative to the magnitude of `x` in `is_constant()` by
@avehtari in #407

Expand Down
20 changes: 11 additions & 9 deletions R/E_loo.R
Original file line number Diff line number Diff line change
Expand Up @@ -271,15 +271,17 @@ E_loo_khat.default <- function(x, psis_object, log_ratios, ...) {
#' @export
E_loo_khat.matrix <- function(x, psis_object, log_ratios, ...) {
tail_lengths <- attr(psis_object, "tail_len")
if (is.null(x)) {
sapply(seq_len(ncol(log_ratios)), function(i) {
.E_loo_khat_i(x, log_ratios[, i], tail_lengths[i])
})
} else {
sapply(seq_len(ncol(log_ratios)), function(i) {
.E_loo_khat_i(x[, i], log_ratios[, i], tail_lengths[i])
})
}
.collect_warnings(
if (is.null(x)) {
sapply(seq_len(ncol(log_ratios)), function(i) {
.E_loo_khat_i(x, log_ratios[, i], tail_lengths[i])
})
} else {
sapply(seq_len(ncol(log_ratios)), function(i) {
.E_loo_khat_i(x[, i], log_ratios[, i], tail_lengths[i])
})
}
)
}

#' Compute function-specific khat estimates
Expand Down
5 changes: 3 additions & 2 deletions R/effective_sample_sizes.R
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ relative_eff.array <- function(x, ..., cores = getOption("mc.cores", 1)) {
S <- prod(dim(x)[1:2]) # posterior sample size = iter * chains

if (cores == 1) {
n_eff_vec <- apply(x, 3, posterior::ess_mean)
n_eff_vec <- .collect_warnings(apply(x, 3, posterior::ess_mean))
} else {
if (!os_is_windows()) {
n_eff_list <-
Expand Down Expand Up @@ -105,14 +105,15 @@ relative_eff.function <-
N <- dim(data)[1]

if (cores == 1) {
n_eff_list <-
n_eff_list <- .collect_warnings(
lapply(
X = seq_len(N),
FUN = function(i) {
val_i <- f_i(data_i = data[i, , drop = FALSE], draws = draws, ...)
relative_eff.default(as.vector(val_i), chain_id = chain_id, cores = 1)
}
)
)
} else {
if (!os_is_windows()) {
n_eff_list <-
Expand Down
29 changes: 29 additions & 0 deletions R/helpers.R
Original file line number Diff line number Diff line change
Expand Up @@ -285,3 +285,32 @@ is_constant <- function(x, tol = .Machine$double.eps) {
# the equality check covers all-zero and all-infinite x
r[1] == r[2] || (is.finite(spread) && spread <= tol * max(abs(r)))
}

#' Collect posterior warnings and signal each class once
#'
#' @noRd
#' @param expr Expression to evaluate.
#' @return The value of `expr`.
.collect_warnings <- function(expr) {
collected <- list()
counts <- list()
on.exit(for (cls in names(collected)) {
cnd <- collected[[cls]]
if (counts[[cls]] > 1) {
cnd$message <- paste0(cnd$message, " (", counts[[cls]], " times)")
}
warning(cnd)
})
withCallingHandlers(
expr,
posterior_warning = function(cnd) {
cls <- class(cnd)[1]
if (is.null(collected[[cls]])) {
collected[[cls]] <<- cnd
counts[[cls]] <<- 0
}
counts[[cls]] <<- counts[[cls]] + 1
invokeRestart("muffleWarning")
}
)
}
Comment thread
florence-bockting marked this conversation as resolved.
Resolved
4 changes: 3 additions & 1 deletion R/loo_moment_matching.R
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,9 @@ loo_moment_match.default <- function(x, loo, post_draws, log_lik_i,
}

if (cores == 1) {
mm_list <- lapply(X = I, FUN = function(i) loo_moment_match_i_fun(i))
mm_list <- .collect_warnings(
lapply(X = I, FUN = function(i) loo_moment_match_i_fun(i))
)
}
else {
if (!os_is_windows()) {
Expand Down
64 changes: 64 additions & 0 deletions tests/testthat/test_0_helpers.R
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,70 @@ test_that("loo_cores works", {
options(loo.cores = NULL)
})

test_that(".collect_warnings() signals each posterior class once", {
Comment thread
florence-bockting marked this conversation as resolved.
Resolved
warn_classed <- function(class) {
warning(structure(
class = c(class, "posterior_warning", "warning", "condition"),
list(message = class, call = NULL)
))
}
caught <- character()
out <- withCallingHandlers(
.collect_warnings({
warn_classed("a")
warn_classed("a")
.collect_warnings(warn_classed("b"))
warn_classed("b")
warning("other")
1
}),
warning = function(cnd) {
caught <<- c(caught, conditionMessage(cnd))
invokeRestart("muffleWarning")
}
)
expect_equal(out, 1)
expect_equal(caught, c("other", "a (2 times)", "b (2 times)"))
})

test_that(".collect_warnings() keeps the first warning and signals it after an error", {
warn_a <- function(msg) {
warning(structure(
class = c("a", "posterior_warning", "warning", "condition"),
list(message = msg, call = NULL)
))
}
w <- capture_warnings(try(.collect_warnings({
warn_a("first")
warn_a("second")
stop("boom")
}), silent = TRUE))
expect_equal(w, "first (2 times)")
})

test_that("relative_eff() signals each posterior warning class once", {
local_mocked_bindings(
ess_mean = function(x, ...) {
warning(structure(
class = c("posterior_warning_x", "posterior_warning", "warning", "condition"),
list(message = "mocked", call = NULL)
))
1
},
.package = "posterior"
)
w <- capture_warnings(relative_eff(exp(example_loglik_array()), cores = 1))
expect_equal(w, "mocked (32 times)")
})

test_that("relative_eff() signals the posterior ESS warning once", {
skip_if_not(packageVersion("posterior") >= "1.7.1")
set.seed(1)
x <- array(rep(c(1, 2), 300) + rnorm(600, 0, 0.1), dim = c(100, 2, 3))
w <- capture_warnings(relative_eff(x, cores = 1))
expect_equal(w, "The ESS has been capped to avoid unstable estimates. (3 times)")
})

test_that("is_constant uses a tolerance relative to the magnitude of x", {
expect_true(is_constant(rep(0, 5)))
expect_true(is_constant(rep(1e-300, 5)))
Expand Down
Loading