From dbc3fee74bb4084ce240581f1c905fb09ece031a Mon Sep 17 00:00:00 2001 From: Sherlock0203 <1297399478@qq.com> Date: Mon, 31 Aug 2026 10:42:30 +0800 Subject: [PATCH 1/9] =?UTF-8?q?[fix]1=E3=80=81unordered=5Fset.emplace=20er?= =?UTF-8?q?ror=202=E3=80=81=E4=BD=BF=E7=94=A8bthread=E4=BB=A3=E6=9B=BF?= =?UTF-8?q?=E5=8E=9F=E6=9C=89timerfd+epoll=20(#33)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/cn/ubring.md | 2 +- docs/en/ubring.md | 2 +- src/brpc/ubshm/common/common.h | 1 + src/brpc/ubshm/shm/shm_ubs.cpp | 78 ++-- src/brpc/ubshm/timer/timer_mgr.cpp | 617 ++++++++++------------------- src/brpc/ubshm/timer/timer_mgr.h | 93 +++-- src/brpc/ubshm/ub_endpoint.cpp | 1 + src/brpc/ubshm/ub_helper.cpp | 5 +- src/brpc/ubshm/ub_ring.cpp | 382 +++++++++++++----- src/brpc/ubshm/ub_ring.h | 8 +- src/brpc/ubshm/ub_ring_manager.cpp | 151 ++++++- src/brpc/ubshm/ub_ring_manager.h | 31 +- src/brpc/ubshm/ubr_trx.h | 68 +++- test/brpc_ubring_unittest.cpp | 3 + 14 files changed, 811 insertions(+), 631 deletions(-) diff --git a/docs/cn/ubring.md b/docs/cn/ubring.md index 1db9909103..cd100db84d 100644 --- a/docs/cn/ubring.md +++ b/docs/cn/ubring.md @@ -187,7 +187,7 @@ UBRing 架构包含以下组件： ### 定时器管理 -UBRing 使用高精度定时器系统 (`timer_mgr.cpp`) 进行连接管理和超时处理，支持 epoll（Linux）和 kqueue（macOS）。 +UBRing 的连接管理和超时处理基于 bthread 定时器（`bthread_timer_add`/`bthread_timer_del`，由 `timer_mgr.cpp` 封装）。非阻塞的 `UbrTimerDel` 可在回调内自删；销毁回调参数所属资源的外部路径用 `UbrTimerDelAndWait` 等待运行中回调退出；一次性定时器触发后自动清理句柄。定时器回调运行在进程全局的 bthread 定时线程上，必须快速返回。关闭检查定时器在链路空闲时按指数退避轮询（上限 `ub_event_queue_timer_interval_max_us`，默认 10ms），有流量或正在关闭时恢复快速轮询。 ## 参考资料 diff --git a/docs/en/ubring.md b/docs/en/ubring.md index fbf87fff36..0cb3d08954 100644 --- a/docs/en/ubring.md +++ b/docs/en/ubring.md @@ -186,7 +186,7 @@ The shared memory manager (`shm_mgr.cpp`) provides a unified interface for diffe ### Timer Management -UBRing uses a high-precision timer system (`timer_mgr.cpp`) for connection management and timeout handling, supporting both epoll (Linux) and kqueue (macOS). +UBRing connection management and timeout handling are built on bthread timers (`bthread_timer_add`/`bthread_timer_del`, wrapped in `timer_mgr.cpp`). The non-blocking `UbrTimerDel` is safe to call from within a timer callback; external teardown that frees resources reachable from the callback argument uses `UbrTimerDelAndWait` to wait out a possibly running callback; one-shot timers clean up their handle automatically when they fire. Timer callbacks run on the process-wide bthread timer thread and must return quickly. The close-check timer backs off exponentially while the link is idle (capped by `ub_event_queue_timer_interval_max_us`, 10ms by default) and returns to fast polling once there is traffic or a close in progress. ## References diff --git a/src/brpc/ubshm/common/common.h b/src/brpc/ubshm/common/common.h index c64ae9843f..504da07c28 100644 --- a/src/brpc/ubshm/common/common.h +++ b/src/brpc/ubshm/common/common.h @@ -116,6 +116,7 @@ static inline int Copy64Byte(int8_t *dst, int8_t *src) { #define SEC_TO_NSEC 1000000000 #define MSEC_TO_NSEC 1000000 +#define SEC_TO_USEC 1000000 #define USEC_TO_NSEC 1000 #define MSEC_TO_SEC 1000 #define MAX_IP_PORT_STR_LEN 23 diff --git a/src/brpc/ubshm/shm/shm_ubs.cpp b/src/brpc/ubshm/shm/shm_ubs.cpp index ccbd325923..a4f27ee1d8 100644 --- a/src/brpc/ubshm/shm/shm_ubs.cpp +++ b/src/brpc/ubshm/shm/shm_ubs.cpp @@ -47,7 +47,7 @@ DEFINE_int32(ub_flying_io_timeout_s, 5, "Time in seconds to wait for stopping data sending and receiving " "when the link is disconnected."); char g_region_name[MAX_REGION_NAME_DESC_LENGTH] = {0}; -int g_shm_timer_fd = 0; +UbrTimerId g_shm_timer_id = nullptr; ShmList *g_shm_list = nullptr; static RETURN_CODE UbsShmInterfacesLoad(void); char hostname[MAX_HOST_NAME_DESC_LENGTH]; @@ -377,14 +377,15 @@ RETURN_CODE UbsShmInit(void) RETURN_CODE UbsShmFini(void) { - int ret = ubsmem_finalize(); - if (ret != UBSM_OK) { - LOG(ERROR) << "Ubs shm finalize fail, ret=" << ret; + // Stop the cleanup timer before finalizing the SDK it calls into. + if (UNLIKELY(DestroyShmTimer(g_shm_list) != UBRING_OK)) { + LOG(ERROR) << "Ubs shm list finalize failed."; return UBRING_ERR; } - if (UNLIKELY(DestroyShmTimer(g_shm_list) != UBRING_OK)) { - LOG(ERROR) << "Ubs shm list finalize failed."; + int ret = ubsmem_finalize(); + if (ret != UBSM_OK) { + LOG(ERROR) << "Ubs shm finalize fail, ret=" << ret; return UBRING_ERR; } @@ -418,50 +419,56 @@ void *UbsShmCallback(void* args) return nullptr; } - LOCK_GUARD(shm_list->shm_lock); - while (shm_list->head != nullptr) { - SHM shm = shm_list->head->shm; - if (shm.addr == nullptr) { - LOG(ERROR) << "Ubs input shm param is invalid, addr is NULL."; + // Drain one node per fire and keep the SDK calls outside the lock, so + // a slow daemon cannot stall the timer thread for the whole list. + SHM shm; + { + LOCK_GUARD(shm_list->shm_lock); + if (shm_list->head == nullptr) { return nullptr; } + shm = shm_list->head->shm; + } + if (shm.addr == nullptr) { + LOG(ERROR) << "Ubs input shm param is invalid, addr is NULL."; + LOCK_GUARD(shm_list->shm_lock); + DeleteShmToList(shm_list); + return nullptr; + } - int ret = ubsmem_shmem_unmap(shm.addr, shm.len); - if (ret != UBSM_OK) { - if (ret == UBSM_ERR_NET) { - return nullptr; - } - LOG(ERROR) << "Ubs unmap shm=" << shm.name << " length=" << shm.len << " failed, ret=" << ret; - return nullptr; + int ret = ubsmem_shmem_unmap(shm.addr, shm.len); + if (ret != UBSM_OK) { + if (ret == UBSM_ERR_NET) { + return nullptr; // retried on the next fire } - LOG(INFO) << "Ubs unmap shm=" << shm.name << " length=" << shm.len << " success."; + LOG(ERROR) << "Ubs unmap shm=" << shm.name << " length=" << shm.len << " failed, ret=" << ret; + return nullptr; // node stays at head, retried + } + LOG(INFO) << "Ubs unmap shm=" << shm.name << " length=" << shm.len << " success."; - ret = ubsmem_shmem_deallocate(shm.name); - if (ret != UBSM_OK) { - DeleteShmToList(shm_list); - LOG(ERROR) << "Ubs delete shm=" << shm.name << " failed, ret=" << ret; - return nullptr; - } + { + LOCK_GUARD(shm_list->shm_lock); DeleteShmToList(shm_list); - LOG(INFO) << "Ubs free local shm=" << shm.name << " length=" << shm.len << " success."; } + ret = ubsmem_shmem_deallocate(shm.name); + if (ret != UBSM_OK) { + LOG(ERROR) << "Ubs delete shm=" << shm.name << " failed, ret=" << ret; + return nullptr; + } + LOG(INFO) << "Ubs free local shm=" << shm.name << " length=" << shm.len << " success."; return nullptr; } RETURN_CODE UbsShmAddTimer(ShmList *shm_list) { - const uint32_t timer_interval_s = FLAGS_ub_flying_io_timeout_s; - itimerspec time_spec = { - .it_interval = {.tv_sec = timer_interval_s, .tv_nsec = 0}, - .it_value = {.tv_sec = 0, .tv_nsec = 1} - }; - int timer_fd = TimerStart(&time_spec, UbsShmCallback, (void*)shm_list); - if (UNLIKELY(timer_fd == -1)) { + const uint64_t timer_interval_us = (uint64_t)FLAGS_ub_flying_io_timeout_s * SEC_TO_USEC; + RETURN_CODE rc = UbrTimerStart(&g_shm_timer_id, 0, timer_interval_us, + UbsShmCallback, (void*)shm_list); + if (UNLIKELY(rc != UBRING_OK)) { LOG(ERROR) << "Start shm timer failed."; return UBRING_ERR; } - g_shm_timer_fd = timer_fd; return UBRING_OK; } @@ -493,7 +500,8 @@ RETURN_CODE InitShmTimer(ShmList **shm_list) RETURN_CODE DestroyShmTimer(ShmList *shm_list) { - DeleteTimerSafe((uint32_t)g_shm_timer_fd); + // Wait out a possibly running UbsShmCallback before tearing shm_list down. + UbrTimerDelAndWait(&g_shm_timer_id); if (shm_list == nullptr) { LOG(WARNING) << "Shm list is null."; return UBRING_ERR; diff --git a/src/brpc/ubshm/timer/timer_mgr.cpp b/src/brpc/ubshm/timer/timer_mgr.cpp index b5e0c9ef3b..c803716a94 100644 --- a/src/brpc/ubshm/timer/timer_mgr.cpp +++ b/src/brpc/ubshm/timer/timer_mgr.cpp @@ -15,454 +15,259 @@ // specific language governing permissions and limitations // under the License. -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include #include -#include +#include +#include "bthread/bthread.h" // bthread_usleep +#include "bthread/unstable.h" // bthread_timer_add/del +#include "butil/time.h" #include "brpc/ubshm/timer/timer_mgr.h" namespace brpc { namespace ubring { -int32_t g_epoll_fd = -1; -std::atomic g_total_timer_num(0); -TimerFdCtx *g_timer_fd_ctx_map = nullptr; -uint32_t g_max_system_fd = 0; -static pthread_t g_epoll_execute_thread = 0; -static int32_t g_timer_module_initialized = 0; - -#if defined(OS_MACOSX) -static int timerfd_create_macosx(int clockid, int flags); -static int timerfd_settime_macosx(int fd, int flags, - const itimerspec *new_value, - itimerspec *old_value); -#endif - -static RETURN_CODE DeleteTimerInner(uint32_t fd) { - if (g_timer_fd_ctx_map == nullptr) { - return UBRING_OK; - } - - if (pthread_spin_lock(&g_timer_fd_ctx_map[fd].spin_lock) != 0) { - return UBRING_ERR; - } - - if (g_timer_fd_ctx_map[fd].status == TIMER_CONTEXT_NOT_USING) { - pthread_spin_unlock(&g_timer_fd_ctx_map[fd].spin_lock); - return UBRING_OK; - } - - g_timer_fd_ctx_map[fd].status = TIMER_CONTEXT_NOT_USING; - g_timer_fd_ctx_map[fd].cb = nullptr; - g_timer_fd_ctx_map[fd].args = nullptr; - g_timer_fd_ctx_map[fd].periodical = 0; - g_timer_fd_ctx_map[fd].fd = 0; - - pthread_spin_unlock(&g_timer_fd_ctx_map[fd].spin_lock); - -#if defined(OS_LINUX) - epoll_ctl(g_epoll_fd, EPOLL_CTL_DEL, (int)fd, nullptr); -#elif defined(OS_MACOSX) - struct kevent evt; - EV_SET(&evt, fd, EVFILT_TIMER, EV_DELETE, 0, 0, nullptr); - kevent(g_epoll_fd, &evt, 1, nullptr, 0, nullptr); -#endif - - uint64_t exp = 0; - read((int)fd, &exp, sizeof(exp)); - - close((int)fd); - std::atomic_fetch_sub(&g_total_timer_num, 1U); - return UBRING_OK; -} - -static RETURN_CODE StartTimeEpoll(void) { -#if defined(OS_LINUX) - g_epoll_fd = epoll_create1(0); -#elif defined(OS_MACOSX) - g_epoll_fd = kqueue(); -#endif - if (UNLIKELY(g_epoll_fd == -1)) { - LOG(ERROR) << "Failed to create epoll/kqueue. errno=" << errno; - return UBRING_ERR; - } - - int ret = pthread_create(&g_epoll_execute_thread, nullptr, TimerEpoll, nullptr); - if (UNLIKELY(ret != 0)) { - LOG(ERROR) << "Failed to create thread err=" << ret; - return UBRING_ERR; - } - return UBRING_OK; -} - -static RETURN_CODE TimerSpinLocksInit(void) { - if (g_timer_fd_ctx_map == nullptr) { - LOG(ERROR) << "Timer module is not fully initialized."; - return UBRING_ERR; - } - - for (uint32_t fd = 0; fd < g_max_system_fd; fd++) { - int ret = pthread_spin_init(&g_timer_fd_ctx_map[fd].spin_lock, - PTHREAD_PROCESS_PRIVATE); - if (ret != EOK) { - LOG(ERROR) << "Failed to initialize spin lock for fd=" << fd; - for (uint32_t cleanup_fd = 0; cleanup_fd < fd; cleanup_fd++) { - pthread_spin_destroy(&g_timer_fd_ctx_map[cleanup_fd].spin_lock); - } - return UBRING_ERR; - } - } - return UBRING_OK; -} - -static RETURN_CODE ExecuteCallback(int32_t timer_fd) { - UnifiedCallback((void *)(&g_timer_fd_ctx_map[timer_fd])); - return UBRING_OK; -} - -static RETURN_CODE TimerCtxMapCompletion(void) { - memset(g_timer_fd_ctx_map, 0, sizeof(TimerFdCtx) * g_max_system_fd); - - RETURN_CODE ret = TimerSpinLocksInit(); - if (ret != UBRING_OK) { - LOG(ERROR) << "Failed to init spin locks for timer module."; - return UBRING_ERR; - } - return UBRING_OK; -} - -RETURN_CODE TimerInit(void) { - if (g_timer_module_initialized > 0) { - return UBRING_OK; - } - - g_total_timer_num.store(0); - - struct rlimit rlim; - if (getrlimit(RLIMIT_NOFILE, &rlim) != UBRING_OK) { - LOG(ERROR) << "Failed to get fd"; - return UBRING_ERR; - } - g_max_system_fd = (uint32_t)rlim.rlim_cur; - - if (g_timer_fd_ctx_map == nullptr) { - g_timer_fd_ctx_map = (TimerFdCtx *)malloc(sizeof(TimerFdCtx) * g_max_system_fd); - if (UNLIKELY(!g_timer_fd_ctx_map)) { - LOG(ERROR) << "Fail to malloc space for timer modules. errno=%d", errno; - return UBRING_ERR; - } - - RETURN_CODE ret = TimerCtxMapCompletion(); - if (ret != UBRING_OK) { - LOG(ERROR) << "Failed to init main data structure of Time Module. ret=" << ret; - free(g_timer_fd_ctx_map); - g_timer_fd_ctx_map = nullptr; - return UBRING_ERR; - } - } - - RETURN_CODE ret = StartTimeEpoll(); - if (ret != UBRING_OK) { - LOG(ERROR) << "Failed to start Timer Epoll. ret=" << ret; - if (LIKELY(g_timer_fd_ctx_map != nullptr)) { - FREE_PTR(g_timer_fd_ctx_map); +namespace { + +enum UbrTimerState { + kStarting = 0, // published, not scheduled yet + kScheduled = 1, + kDead = 2 // scheduling failed +}; + +} // namespace + +// Reference rules: one "owner" ref for the handle slot, one "schedule" ref +// per pending/running bthread schedule, plus one ref held by the starter +// until its post-schedule bookkeeping is done. The schedule ref is +// consumed by the firing callback or by the deleter whose +// bthread_timer_del returned 0 (cancelled before run); the owner ref is +// consumed by whoever takes the task out of *slot -- a deleter, or the +// one-shot firing callback itself, which exits the slot BEFORE running +// the callback so that the callback may free the object storing the slot. +// All atomics are seq_cst so no interleaving can release a ref twice or +// free the task while a callback or the starter still touches it. +struct UbrTimerTask { + UbrTimerId* slot; + std::atomic id; + void* (*cb)(void*); + void* arg; + UbrTimerBackoffFn backoff; + uint64_t interval_us; // timer thread only + bool periodic; + std::atomic state; // kStarting/kScheduled/kDead + std::atomic stopped; + std::atomic ref; + std::atomic join_pending; // a DelAndWait is waiting + std::atomic done; // refs hit zero, joiner frees +}; + +namespace { + +void ReleaseRef(UbrTimerTask* task) { + if (task->ref.fetch_sub(1) == 1) { + if (task->join_pending.load()) { + task->done.store(true); // joiner frees the task + } else { + delete task; } - return UBRING_ERR; - } - g_timer_module_initialized = 1; - return UBRING_OK; -} - -void *UnifiedCallback(void *args) { - TimerFdCtx *ctx = (TimerFdCtx *)args; - if (pthread_spin_lock(&ctx->spin_lock) != 0) { - return nullptr; - } - - if (ctx->status == TIMER_CONTEXT_NOT_USING) { - pthread_spin_unlock(&ctx->spin_lock); - return nullptr; - } - - void *(*cb)(void *) = ctx->cb; - void *cb_args = ctx->args; - uint32_t fd = ctx->fd; - int is_periodical = ctx->periodical; - ctx->status = TIMER_CONTEXT_CALLBACK_ONGOING; - - pthread_spin_unlock(&ctx->spin_lock); - - cb(cb_args); - - if (!is_periodical) { - DeleteTimerInner(fd); } - return nullptr; } -void *TimerEpoll(void *args) { - UNREFERENCE_PARAM(args); -#if defined(OS_LINUX) - struct epoll_event ready_events[MAX_TIMER]; -#elif defined(OS_MACOSX) - struct kevent ready_events[MAX_TIMER]; -#endif +void UbrTimerOnFire(void* p) { + UbrTimerTask* task = (UbrTimerTask*)p; - while (1) { - if (g_timer_module_initialized <= 0) { - LOG(ERROR) << "The Timer module is not initialized."; - break; + if (task->periodic) { + if (!task->stopped.load()) { + task->cb(task->arg); } - -#if defined(OS_LINUX) - int32_t ready_num = epoll_wait(g_epoll_fd, ready_events, MAX_TIMER, - TIMER_EPOLL_WAIT_TIMEOUT); -#elif defined(OS_MACOSX) - struct timespec timeout = {0, TIMER_EPOLL_WAIT_TIMEOUT * 1000000}; - int32_t ready_num = kevent(g_epoll_fd, nullptr, 0, ready_events, MAX_TIMER, &timeout); -#endif - - if (UNLIKELY(ready_num == -1)) { - errno_t err = errno; - if (err == EINTR) { - LOG_EVERY_SECOND(WARNING) << "Epoll/Kqueue wait was interrupted. errno=" << err; - continue; - } else if (err == EBADF) { - LOG(WARNING) << "The Timer module is destroyed."; - break; + // Claim the next schedule's ref before re-reading `stopped' so a + // racing delete can neither free the task nor orphan a re-arm. + task->ref.fetch_add(1); + if (task->stopped.load()) { + ReleaseRef(task); + } else { + uint64_t interval = task->interval_us; + if (task->backoff != nullptr) { + interval = task->backoff(task->arg, interval); + task->interval_us = interval; } - LOG(ERROR) << "Epoll/Kqueue wait internal error. errno=" << err; - break; - } - - for (int32_t i = 0; i < ready_num; i++) { -#if defined(OS_LINUX) - struct epoll_event *event = &ready_events[i]; - int32_t timer_fd = event->data.fd; -#elif defined(OS_MACOSX) - struct kevent *event = &ready_events[i]; - int32_t timer_fd = event->ident; -#endif - - uint64_t exp = 0; - if (read(timer_fd, &exp, sizeof(exp)) < 0) { - if (errno != EBADF) { - LOG(ERROR) << "Failed to read timerfd=" << timer_fd << " errno=" << errno; + bthread_timer_t id = 0; + if (bthread_timer_add( + &id, butil::microseconds_from_now((int64_t)interval), + UbrTimerOnFire, task) == 0) { + task->id.store(id); + if (task->stopped.load() && bthread_timer_del(id) == 0) { + ReleaseRef(task); } - continue; - } - if (TimerFdCtxValidate((uint32_t)timer_fd) != UBRING_OK) { - continue; - } - - RETURN_CODE ret = ExecuteCallback(timer_fd); - if (ret != UBRING_OK) { - LOG(ERROR) << "Failed execute callback ret=" << ret; - DeleteTimerInner((uint32_t)timer_fd); - continue; + } else { + LOG(ERROR) << "Fail to re-arm ubring timer"; + ReleaseRef(task); } } - } - return nullptr; -} - -void DeleteTimerSafe(uint32_t fd) { - if (g_timer_fd_ctx_map == nullptr) { + ReleaseRef(task); return; } - if (pthread_spin_lock(&g_timer_fd_ctx_map[fd].spin_lock) != 0) { - return; + // One-shot: exit the handle slot first -- after this the wrapper never + // touches the storage again, so the callback may release the object + // that holds it. Whether the callback runs is decided solely by this + // slot competition: every UbrTimerDel that wants the callback + // suppressed has to win this exchange first, so owned==true guarantees + // no UbrTimerDel is pending. Do not consult `stopped' here: its store + // (del thread) and this load (timer thread) are separated by the slot + // RMW and seq_cst does not order the store-buffer case -- ownership of + // the slot is the single arbiter. + UbrTimerId expected = task; + const bool owned = + __atomic_compare_exchange_n(task->slot, &expected, (UbrTimerId) nullptr, + false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); + if (owned) { + task->cb(task->arg); + } + ReleaseRef(task); // schedule + if (owned) { + ReleaseRef(task); // owner } - - if (g_timer_fd_ctx_map[fd].status == TIMER_CONTEXT_NOT_USING) { - pthread_spin_unlock(&g_timer_fd_ctx_map[fd].spin_lock); - return; - } - - g_timer_fd_ctx_map[fd].status = TIMER_CONTEXT_NOT_USING; - g_timer_fd_ctx_map[fd].cb = nullptr; - g_timer_fd_ctx_map[fd].args = nullptr; - g_timer_fd_ctx_map[fd].periodical = 0; - g_timer_fd_ctx_map[fd].fd = 0; - - pthread_spin_unlock(&g_timer_fd_ctx_map[fd].spin_lock); - -#if defined(OS_LINUX) - epoll_ctl(g_epoll_fd, EPOLL_CTL_DEL, (int)fd, nullptr); -#elif defined(OS_MACOSX) - struct kevent evt; - EV_SET(&evt, fd, EVFILT_TIMER, EV_DELETE, 0, 0, nullptr); - kevent(g_epoll_fd, &evt, 1, nullptr, 0, nullptr); -#endif - - uint64_t exp = 0; - read((int)fd, &exp, sizeof(exp)); - - close((int)fd); - std::atomic_fetch_sub(&g_total_timer_num, 1U); } -void DeleteTimer(uint32_t fd) { - if (g_timer_fd_ctx_map == nullptr) { - LOG(WARNING) << "The timer is not initialized."; - return; - } - - g_timer_fd_ctx_map[fd].periodical = 0; +UbrTimerTask* TakeOutTask(UbrTimerId* slot) { + return __atomic_exchange_n(slot, (UbrTimerId) nullptr, __ATOMIC_SEQ_CST); } -int32_t TimerStart(const itimerspec *time, void *(*cb)(void *), void *args) { - if (g_epoll_fd == -1) { - LOG(ERROR) << "Timer epoll/kqueue encountered internal error."; - return -1; - } - -#if defined(OS_LINUX) - int timer_fd = timerfd_create(CLOCK_MONOTONIC, 0); -#elif defined(OS_MACOSX) - int timer_fd = timerfd_create_macosx(CLOCK_MONOTONIC, 0); -#endif - - if (UNLIKELY(timer_fd >= (int)g_max_system_fd || timer_fd == -1)) { - LOG(ERROR) << "Failed to create timerfd=" << timer_fd << " errno=" << errno; - return -1; +RETURN_CODE TimerStartInternal(UbrTimerId* slot, uint64_t delay_us, + uint64_t interval_us, void* (*cb)(void*), + void* arg, UbrTimerBackoffFn backoff) { + if (UNLIKELY(slot == nullptr || cb == nullptr)) { + LOG(ERROR) << "Ubr timer start invalid argument, slot=" << slot; + return UBRING_ERR; } - g_timer_fd_ctx_map[timer_fd].status = TIMER_CONTEXT_EPOLL_WAITING; - g_timer_fd_ctx_map[timer_fd].cb = cb; - g_timer_fd_ctx_map[timer_fd].args = args; - g_timer_fd_ctx_map[timer_fd].fd = (uint32_t)timer_fd; - - if (LIKELY(time->it_interval.tv_sec > 0 || time->it_interval.tv_nsec > 0)) { - g_timer_fd_ctx_map[timer_fd].periodical = 1; + UbrTimerTask* task = new (std::nothrow) UbrTimerTask(); + if (UNLIKELY(task == nullptr)) { + LOG(ERROR) << "Fail to malloc ubring timer task."; + return UBRING_ERR; } - -#if defined(OS_LINUX) - struct epoll_event event = { - .events = EPOLLIN, - .data = {.fd = timer_fd} - }; - - int32_t ret = epoll_ctl(g_epoll_fd, EPOLL_CTL_ADD, timer_fd, &event); -#elif defined(OS_MACOSX) - struct kevent event; - uint64_t timeout_nsec = time->it_value.tv_sec * 1000000000ULL + time->it_value.tv_nsec; - uint64_t interval_nsec = time->it_interval.tv_sec * 1000000000ULL + time->it_interval.tv_nsec; - EV_SET(&event, timer_fd, EVFILT_TIMER, EV_ADD | EV_ENABLE, 0, - timeout_nsec / 1000000, nullptr); - int32_t ret = kevent(g_epoll_fd, &event, 1, nullptr, 0, nullptr); -#endif - - if (UNLIKELY(ret != 0)) { - CloseTimerFd(timer_fd); - LOG(ERROR) << "Failed to add event to epoll/kqueue. errno=" << errno; - return -1; + task->slot = slot; + task->id.store(0); + task->cb = cb; + task->arg = arg; + task->backoff = backoff; + task->interval_us = interval_us; + task->periodic = (interval_us > 0); + task->state.store(kStarting); + task->stopped.store(false); + task->ref.store(3); // owner + schedule + starter + task->join_pending.store(false); + task->done.store(false); + + // Publish the real task before scheduling so a delete or a DelAndWait + // racing the start always has an object to act on or wait for. + UbrTimerId expected = nullptr; + if (!__atomic_compare_exchange_n(slot, &expected, (UbrTimerId) task, false, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) { + LOG(ERROR) << "Ubr timer start refused, slot already occupied"; + delete task; // never published + return UBRING_ERR; } - std::atomic_fetch_add(&g_total_timer_num, 1U); - -#if defined(OS_LINUX) - ret = timerfd_settime(timer_fd, 0, time, nullptr); -#elif defined(OS_MACOSX) - ret = timerfd_settime_macosx(timer_fd, 0, time, nullptr); -#endif - - if (UNLIKELY(ret != 0)) { -#if defined(OS_LINUX) - if (epoll_ctl(g_epoll_fd, EPOLL_CTL_DEL, timer_fd, nullptr) != 0) { -#elif defined(OS_MACOSX) - struct kevent evt; - EV_SET(&evt, timer_fd, EVFILT_TIMER, EV_DELETE, 0, 0, nullptr); - if (kevent(g_epoll_fd, &evt, 1, nullptr, 0, nullptr) != 0) { -#endif - LOG(ERROR) << "Failed to delete the timer fd=" << timer_fd << " with errno=" << errno; + bthread_timer_t id = 0; + if (UNLIKELY(bthread_timer_add( + &id, butil::microseconds_from_now((int64_t)delay_us), + UbrTimerOnFire, task) != 0)) { + LOG(ERROR) << "Fail to add ubring timer"; + task->state.store(kDead); // wake DelAndWait waiters + expected = task; + const bool owned = + __atomic_compare_exchange_n(slot, &expected, (UbrTimerId) nullptr, + false, __ATOMIC_SEQ_CST, + __ATOMIC_SEQ_CST); + ReleaseRef(task); // schedule, never ran + if (owned) { + ReleaseRef(task); // owner } - CloseTimerFd(timer_fd); - std::atomic_fetch_sub(&g_total_timer_num, 1U); - LOG(ERROR) << "Failed to set timer"; - return -1; + ReleaseRef(task); // starter + return UBRING_ERR; } - - return timer_fd; + // A zero-delay task may have fired and re-armed already; keep a newer + // id if so. + bthread_timer_t expected_id = 0; + task->id.compare_exchange_strong(expected_id, id); + task->state.store(kScheduled); + // No post-add stopped check here: a UbrTimerDel racing the start + // returns 1 without consuming the per-task resources, and the armed + // timer must fire so that OnFire settles the ownership protocol. + ReleaseRef(task); // starter + return UBRING_OK; } -uint32_t GetActiveTimerNum(void) { - return std::atomic_load(&g_total_timer_num); -} +} // namespace -void CloseTimerFd(int fd) { - g_timer_fd_ctx_map[fd].cb = nullptr; - g_timer_fd_ctx_map[fd].args = nullptr; - g_timer_fd_ctx_map[fd].status = TIMER_CONTEXT_NOT_USING; - g_timer_fd_ctx_map[fd].fd = 0; - g_timer_fd_ctx_map[fd].periodical = 0; - if (close((int)fd) != 0) { - LOG(ERROR) << "Failed to close timer fd=" << fd << " errno=" << errno; - return; - } +RETURN_CODE UbrTimerStart(UbrTimerId* slot, uint64_t delay_us, + uint64_t interval_us, void* (*cb)(void*), + void* arg, UbrTimerBackoffFn backoff) { + return TimerStartInternal(slot, delay_us, interval_us, cb, arg, backoff); } -void TimerModuleDestroy(void) { - uint32_t max_fd = g_max_system_fd; - if (g_timer_fd_ctx_map) { - for (uint32_t fd = 0; fd < max_fd; fd++) { - if (g_timer_fd_ctx_map[fd].status != TIMER_CONTEXT_NOT_USING) { - DeleteTimerSafe(fd); - } - } - } - close(g_epoll_fd); - g_epoll_fd = -1; - g_total_timer_num = 0; - g_timer_module_initialized = 0; - int32_t ret = pthread_join(g_epoll_execute_thread, nullptr); - if (ret != EOK) { - LOG(ERROR) << "Failed to join pthread, during destroying timer module. ret=" << ret; - return; - } +int UbrTimerDel(UbrTimerId* slot) { + if (slot == nullptr) { + return 1; + } + // Take the ownership of the slot first: after this exchange every + // dereference below is safe (the task cannot be freed while we hold + // the owner reference the slot used to anchor). + UbrTimerTask* task = TakeOutTask(slot); + if (task == nullptr) { + return 1; // fired and cleared its slot (callback side consumed) + // or another del won the exchange (it consumes) + } + task->stopped.store(true); // meaningful for periodic only + // A start still in flight cannot be cancelled nor dispatched yet; wait + // for the starter to settle the fate (kScheduled/kDead). Bounded: the + // starter stores the state before taking any lock our caller holds. + while (task->state.load() == kStarting) { + bthread_usleep(1000); + } + if (task->state.load() == kDead) { + ReleaseRef(task); // owner; schedule/starter are + return 1; // settled by the kDead path + } + bthread_timer_t id = task->id.load(); + if (id != 0 && bthread_timer_del(id) == 0) { + ReleaseRef(task); // schedule: cancelled before dispatch + } // ==1: dispatched, OnFire (owned==false) + // releases it + ReleaseRef(task); // owner + return 0; // the callback is guaranteed never to run: the caller + // consumes the timer/callback reference } -RETURN_CODE TimerFdCtxValidate(uint32_t fd) { - if (fd >= g_max_system_fd) { - LOG(ERROR) << "TimerFd=" << fd << " is out of range=" << g_max_system_fd; - return UBRING_ERR; - } - if (g_timer_fd_ctx_map[fd].status == TIMER_CONTEXT_NOT_USING) { - LOG(ERROR) << "TimerFd=" << fd << " has wrong status=" << g_timer_fd_ctx_map[fd].status; - return UBRING_ERR; +void UbrTimerDelAndWait(UbrTimerId* slot) { + if (slot == nullptr) { + return; } - if (g_timer_fd_ctx_map[fd].cb == nullptr) { - LOG(ERROR) << "The callback is not set."; - return UBRING_ERR; + UbrTimerTask* task = TakeOutTask(slot); + if (task == nullptr) { + return; } - - return UBRING_OK; -} - -#if defined(OS_MACOSX) -static int timerfd_create_macosx(int clockid, int flags) { - int pipefd[2]; - if (pipe(pipefd) == -1) { - return -1; + task->join_pending.store(true); + task->stopped.store(true); + // A start still in flight cannot be cancelled yet; wait for the + // starter to schedule it or mark it dead. + while (task->state.load() == kStarting) { + bthread_usleep(1000); + } + if (task->state.load() == kScheduled) { + bthread_timer_t id = task->id.load(); + if (id != 0 && bthread_timer_del(id) == 0) { + ReleaseRef(task); // cancelled before run + } } - return pipefd[0]; -} - -static int timerfd_settime_macosx(int fd, int flags, - const itimerspec *new_value, - itimerspec *old_value) { - if (old_value != nullptr) { - memset(old_value, 0, sizeof(itimerspec)); + ReleaseRef(task); // owner reference + while (!task->done.load()) { + bthread_usleep(1000); } - return 0; + task->join_pending.store(false); + delete task; } -#endif } // namespace ubring -} // namespace brpc \ No newline at end of file +} // namespace brpc diff --git a/src/brpc/ubshm/timer/timer_mgr.h b/src/brpc/ubshm/timer/timer_mgr.h index 1b42caef04..74e5083ce7 100644 --- a/src/brpc/ubshm/timer/timer_mgr.h +++ b/src/brpc/ubshm/timer/timer_mgr.h @@ -15,59 +15,58 @@ // specific language governing permissions and limitations // under the License. +// bthread based timer facade for the ubring module. Callbacks run on the +// process-wide bthread timer thread and must return quickly. + #ifndef BRPC_TIMER_MGR_H #define BRPC_TIMER_MGR_H -#include -#include -#include "brpc/ubshm/common/common.h" -#if defined(OS_LINUX) -#include -#include -#elif defined(OS_MACOSX) -#include -#include -#include -#endif - -#define MAX_TIMER 1024 -#define TIMER_EPOLL_WAIT_TIMEOUT 1000 +#include +#include "brpc/ubshm/common/common.h" -#if defined(OS_MACOSX) -struct itimerspec -{ - struct timespec it_interval; - struct timespec it_value; -}; -#endif namespace brpc { namespace ubring { -typedef enum { - TIMER_CONTEXT_NOT_USING, - TIMER_CONTEXT_EPOLL_WAITING, - TIMER_CONTEXT_CALLBACK_ONGOING -} TimerFdCtxStatus; -typedef struct { - void *(*cb)(void*); - void *args; - uint32_t fd; - TimerFdCtxStatus status; - uint32_t periodical; - pthread_spinlock_t spin_lock; -} TimerFdCtx; +// Opaque timer handle. nullptr means "not started" (or already deleted / +// fired for one-shot timers). +typedef struct UbrTimerTask* UbrTimerId; + +// Maps the current re-arm interval of a periodic timer to the next one. +// Runs on the timer thread only. +typedef uint64_t (*UbrTimerBackoffFn)(void* arg, uint64_t cur_interval_us); + +// Schedule `cb(arg)' to run after `delay_us' and, when `interval_us' > 0, +// re-arm itself after every run until deleted. One-shot timers release +// their handle slot before running the callback, so the callback may free +// the object that stores the slot; the task object itself is released +// automatically. +RETURN_CODE UbrTimerStart(UbrTimerId* slot, uint64_t delay_us, + uint64_t interval_us, void* (*cb)(void*), + void* arg, + UbrTimerBackoffFn backoff = nullptr); + +// Non-blocking delete, safe from inside the timer callback itself. Does +// not wait for a running callback and does not protect `arg' on its own. +// Returns 0 when the call won the slot competition: a one-shot callback +// is guaranteed never to run, and the caller consumes any per-task +// resources it tracks for this timer (ownership of them transfers to the +// caller); for a periodic timer an already-started callback is not +// interrupted. Returns 1 when the callback has been dispatched (it +// consumes those resources itself on every exit) or its fate is still +// being settled by the scheduler -- the caller must not consume anything +// then. +int UbrTimerDel(UbrTimerId* slot); + +// Delete and wait until a possibly running callback finished, so the +// caller can free resources reachable from `arg'. Never call this on the +// callback's own task. A one-shot callback that is already running holds +// the ownership of `arg' by itself (mirroring bthread_timer_del returning +// 1) and cannot be waited for through the slot. The wait polls with +// bthread_usleep, which degrades to ::usleep on plain pthread callers +// (e.g. process-exit paths). +void UbrTimerDelAndWait(UbrTimerId* slot); -RETURN_CODE TimerInit(void); -void TimerModuleDestroy(void); -void *UnifiedCallback(void *args); -void *TimerEpoll(void *args); -int32_t TimerStart(const itimerspec *time, void *(*cb)(void *), void *args); -uint32_t GetActiveTimerNum(void); -void CloseTimerFd(int fd); +} // namespace ubring +} // namespace brpc -void DeleteTimerSafe(uint32_t fd); -void DeleteTimer(uint32_t fd); -RETURN_CODE TimerFdCtxValidate(uint32_t fd); -} -} -#endif //BRPC_TIMER_MGR_H \ No newline at end of file +#endif //BRPC_TIMER_MGR_H diff --git a/src/brpc/ubshm/ub_endpoint.cpp b/src/brpc/ubshm/ub_endpoint.cpp index 31539fda85..089ccef5bd 100644 --- a/src/brpc/ubshm/ub_endpoint.cpp +++ b/src/brpc/ubshm/ub_endpoint.cpp @@ -848,6 +848,7 @@ int UBShmEndpoint::PollingModeInitialize(bthread_tag_t tag, while (running->load(std::memory_order_relaxed)) { while (poller->op_queue.Dequeue(op)) { if (op.type == PollerSidOp::ADD) { + poller_sids.erase(op); poller_sids.emplace(op); } else if (op.type == PollerSidOp::REMOVE) { poller_sids.erase(op); diff --git a/src/brpc/ubshm/ub_helper.cpp b/src/brpc/ubshm/ub_helper.cpp index 230ca71bac..d3c769140d 100644 --- a/src/brpc/ubshm/ub_helper.cpp +++ b/src/brpc/ubshm/ub_helper.cpp @@ -59,10 +59,7 @@ static void GlobalUBInitializeOrDieImpl() { ExitWithError(); } - if (TimerInit()) { - PLOG(ERROR) << "Fail to TimerInit"; - ExitWithError(); - } + // Timers are bthread based and start lazily on first use. if (ShmMgrInit()) { PLOG(ERROR) << "Fail to ShmMgrInit"; diff --git a/src/brpc/ubshm/ub_ring.cpp b/src/brpc/ubshm/ub_ring.cpp index cdc346edf4..70db4f59cb 100644 --- a/src/brpc/ubshm/ub_ring.cpp +++ b/src/brpc/ubshm/ub_ring.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include "bthread/bthread.h" #include "butil/logging.h" #include "brpc/ubshm/ub_ring.h" @@ -38,6 +39,14 @@ DEFINE_int32(ub_hb_retry_cnt, 10, "UBRing heartbeat retry count."); DEFINE_int32(ub_event_queue_timer_interval_us, 100, "UBRing disconnection check interval in microseconds."); +DEFINE_int32(ub_event_queue_timer_interval_max_us, 10000, + "UBRing upper bound of the close-check polling interval in " + "microseconds while the link is idle; the interval backs off " + "from ub_event_queue_timer_interval_us up to this value. " + "Set to 0 to keep the interval steady (back-off disabled)."); + +// Exponential back-off multiplier of the close-check polling interval. +constexpr uint64_t kCloseCheckBackoffFactor = 2; UBRing::UBRing() {} @@ -59,6 +68,122 @@ RETURN_CODE UBRing::UbrTrxMapShm(SHM *local_shm, SHM *remote_shm) return UBRING_OK; } +static void UbrDoAsynClearWork(UbrTrx *trx, uint64_t expect_ubr_id) { + if (UNLIKELY(UBRing::UbrTrxFreeShm(trx) != UBRING_OK)) { + LOG(ERROR) << "Trx close, wait for local shm " << trx->local_shm.name << " free fail."; + } + if (UNLIKELY(UBRingManager::ReleaseUbrTrxFromMgr(trx, expect_ubr_id) != UBRING_OK)) { + LOG(ERROR) << "Trx close, release shm " << trx->local_shm.name << " trx failed."; + } +} + +static void UbrDoPassiveClearWork(UbrTrx *trx, uint64_t expect_ubr_id) { + int rc = ShmLocalFree(&trx->remote_shm); + if (rc != UBRING_OK) { + LOG(ERROR) << "Trx passive clear, delete remote shm " << trx->remote_shm.name + << " failed. ret=" << rc; + } + rc = ShmLocalFree(&trx->local_shm); + if (rc != UBRING_OK) { + LOG(ERROR) << "Trx passive clear, delete local shm " << trx->local_shm.name + << " failed. ret=" << rc; + } + if (UNLIKELY(UBRingManager::ReleaseUbrTrxFromMgr(trx, expect_ubr_id) != UBRING_OK)) { + LOG(ERROR) << "Trx passive clear, release shm " << trx->local_shm.name << " trx failed."; + } +} + +// Schedule the delayed cleanup of `trx'. The cleanup ownership lives in the +// per-acquisition control object, so exactly one of the delayed-clear +// callback and a force close ever runs the cleanup. `work' is the cleanup +// body, used directly when the timer cannot be started. +static RETURN_CODE UbrScheduleClearTimer(UbrTrx *trx, void* (*cb)(void*), + void (*work)(UbrTrx*, uint64_t)) { + if (UNLIKELY(trx == nullptr || trx->local_shm.addr == nullptr)) { + return UBRING_OK; // released trx, stale event + } + if (__atomic_load_n(&trx->cleanup_ctl, __ATOMIC_SEQ_CST) != nullptr) { + return UBRING_OK; // cleanup already scheduled + } + auto* ctl = new (std::nothrow) UbrCleanupCtl(); + if (UNLIKELY(ctl == nullptr)) { + LOG(ERROR) << "Fail to malloc ubr cleanup ctl."; + return UBRING_ERR; + } + ctl->trx = trx; + ctl->ubr_id = ATOMIC_LOAD(trx->ubr_id); + ctl->state.store(UBR_CLEANUP_PENDING); + ctl->timer = nullptr; + ctl->ref.store(2); // timer/callback + starter; the + // manager anchor is taken by + // TryPublishUnitCleanupCtl + + UbrCleanupCtl* expected = nullptr; + if (!__atomic_compare_exchange_n(&trx->cleanup_ctl, &expected, ctl, false, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) { + delete ctl; // another schedule won + return UBRING_OK; + } + if (!UBRingManager::TryPublishUnitCleanupCtl(trx->trx_mgr_index, + ctl->ubr_id, ctl)) { + // The slot was released (and possibly reused) before we could + // anchor: force close or the new occupant owns it now. Nothing + // was armed yet -- just undo the trx-side publication. + expected = ctl; + __atomic_compare_exchange_n(&trx->cleanup_ctl, &expected, + (UbrCleanupCtl*) nullptr, false, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); + ctl->ReleaseRef(); // timer/callback reference, never armed + ctl->ReleaseRef(); // starter reference + return UBRING_OK; + } + RETURN_CODE rc = UbrTimerStart(&ctl->timer, + (uint64_t)FLAGS_ub_flying_io_timeout_s * SEC_TO_USEC, 0, cb, ctl); + if (UNLIKELY(rc != UBRING_OK)) { + // The timer was never scheduled: this path owns the manager, + // timer/callback and starter references. Roll the schedule back + // and run the cleanup inline so the trx does not end up with + // neither timers nor a queued cleanup. If force close claimed the + // ownership meanwhile, leave the manager anchor to it. + int state_expected = UBR_CLEANUP_PENDING; + if (ATOMIC_COMPARE_EXCHANGE_STRONG(ctl->state, state_expected, UBR_CLEANUP_RUNNING)) { + if (ATOMIC_LOAD(trx->ubr_id) == ctl->ubr_id) { + work(trx, ctl->ubr_id); + } + ATOMIC_STORE(ctl->state, UBR_CLEANUP_DONE); + // Detach only after DONE: while the inline cleanup runs, the + // anchor and cleanup_ctl must keep telling a concurrent force + // close that this cleanup is owned (RUNNING/DONE), otherwise it + // would fall into its no-ctl branch and clean the trx again. + UbrCleanupCtl* published = ctl; + __atomic_compare_exchange_n(&trx->cleanup_ctl, &published, + (UbrCleanupCtl*) nullptr, false, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); + UBRingManager::DetachUnitCleanupCtl(trx->trx_mgr_index, ctl); + } + ctl->ReleaseRef(); // timer/callback reference + ctl->ReleaseRef(); // starter reference + return UBRING_ERR; + } + if (UNLIKELY(ATOMIC_LOAD(trx->ubr_id) != ctl->ubr_id)) { + // Published onto a slot that was released and reused meanwhile. + if (UbrTimerDel(&ctl->timer) == 0) { + ctl->ReleaseRef(); // timer/callback reference + } + UbrCleanupCtl* published = ctl; + if (!__atomic_compare_exchange_n(&trx->cleanup_ctl, &published, + (UbrCleanupCtl*) nullptr, false, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) { + published = nullptr; + } + UBRingManager::DetachUnitCleanupCtl(trx->trx_mgr_index, ctl); + ctl->ReleaseRef(); // starter reference + return UBRING_OK; + } + ctl->ReleaseRef(); // starter reference + return UBRING_OK; +} + RETURN_CODE UBRing::UbrTrxClose() { RETURN_CODE close_check_rc = UbrTrxCloseCheck(_trx); if (UNLIKELY(close_check_rc != UBRING_OK)) { @@ -89,17 +214,46 @@ RETURN_CODE UBRing::UbrTrxClose() { LOG(WARNING) << "Local shm " << _trx->local_shm.name << " wait for the peer to close timed out, force cleanup."; _trx->ubr_rx.trx_state = UBR_STATE_CLOSED; - // Force synchronous cleanup instead of relying on async timer - DeleteTimerSafe((uint32_t)_trx->timer_fd); - DeleteTimerSafe((uint32_t)_trx->hb_timer_fd); - if (_trx->ubr_tx.remote_rx_event_q.addr != nullptr) { - ((UbrEventQMsg *)_trx->ubr_tx.remote_rx_event_q.addr)->flag = UBR_STATE_CLOSED; + // Wait out the close/heartbeat callbacks, which may schedule a + // delayed cleanup, then settle the cleanup ownership: force + // runs the cleanup itself when it can claim it, and leaves it + // to an already running delayed-clear callback otherwise. + const uint64_t expect_ubr_id = ATOMIC_LOAD(_trx->ubr_id); + UbrTimerDelAndWait(&_trx->close_timer); + UbrTimerDelAndWait(&_trx->hb_timer); + UbrCleanupCtl* ctl = UBRingManager::SnapshotUnitCleanupCtl(_trx->trx_mgr_index); + if (ctl != nullptr && ctl->ubr_id != expect_ubr_id) { + ctl->ReleaseRef(); // snapshot reference + ctl = nullptr; // slot reused, not ours + } + bool cleanup_owned = false; + if (ctl != nullptr) { + int expected = UBR_CLEANUP_PENDING; + if (ATOMIC_COMPARE_EXCHANGE_STRONG(ctl->state, expected, UBR_CLEANUP_RUNNING)) { + cleanup_owned = true; + if (UbrTimerDel(&ctl->timer) == 0) { + ctl->ReleaseRef(); // timer/callback reference + } + } + } else if (ATOMIC_LOAD(_trx->ubr_id) == expect_ubr_id) { + cleanup_owned = true; } - if (UNLIKELY(UbrTrxFreeShm(_trx) != UBRING_OK)) { - LOG(WARNING) << "Force close, local shm " << _trx->local_shm.name << " free failed."; + if (cleanup_owned) { + if (_trx->ubr_tx.remote_rx_event_q.addr != nullptr) { + ((UbrEventQMsg *)_trx->ubr_tx.remote_rx_event_q.addr)->flag = UBR_STATE_CLOSED; + } + if (UNLIKELY(UbrTrxFreeShm(_trx) != UBRING_OK)) { + LOG(WARNING) << "Force close, local shm " << _trx->local_shm.name << " free failed."; + } + if (UNLIKELY(UBRingManager::ReleaseUbrTrxFromMgr(_trx, expect_ubr_id) != UBRING_OK)) { + LOG(WARNING) << "Force close, release trx " << _trx->local_shm.name << " failed."; + } + if (ctl != nullptr) { + ATOMIC_STORE(ctl->state, UBR_CLEANUP_DONE); + } } - if (UNLIKELY(UBRingManager::ReleaseUbrTrxFromMgr(_trx) != UBRING_OK)) { - LOG(WARNING) << "Force close, release trx " << _trx->local_shm.name << " failed."; + if (ctl != nullptr) { + ctl->ReleaseRef(); // snapshot reference } return UBRING_ERR_TIMEOUT; } @@ -107,7 +261,7 @@ RETURN_CODE UBRing::UbrTrxClose() { } _trx->ubr_rx.trx_state = UBR_STATE_CLOSED; RETURN_CODE rc; - if (UNLIKELY((rc = ClearTrxResource(_trx, start_time, UBR_SEND_CLOSE)) != UBRING_OK)) { + if (UNLIKELY((rc = ClearTrxResource(_trx)) != UBRING_OK)) { if (rc == UBRING_REENTRY) { LOG(INFO) << "Trx close, peer is closing, trx local name=" << _trx->local_shm.name; return UBRING_OK; @@ -124,24 +278,51 @@ RETURN_CODE UBRing::UbrTrxClose() { return UBRING_OK; } +// Back-off policy of the close-check timer: fast while there is traffic or +// a close in progress, doubling up to the cap while idle. +static uint64_t UbrCloseTimerBackoff(void* arg, uint64_t cur_interval_us) { + auto* trx = (UbrTrx*)arg; + auto* local_rx_event_q = (UbrEventQMsg *)trx->ubr_rx.local_rx_event_q.addr; + auto* local_tx_event_q = (UbrEventQMsg *)trx->ubr_tx.local_tx_event_q.addr; + const uint64_t in_io_id = ATOMIC_LOAD(trx->ubr_rx.in_io_id); + const uint64_t out_io_id = ATOMIC_LOAD(trx->ubr_tx.out_io_id); + if (UNLIKELY(local_rx_event_q == nullptr)) { + return (uint64_t)FLAGS_ub_event_queue_timer_interval_us; + } + const bool has_traffic = (in_io_id != trx->close_chk_in_io_id) || + (out_io_id != trx->close_chk_out_io_id); + const bool closing = (local_rx_event_q->flag != UBR_STATE_CONNECTED); + trx->close_chk_in_io_id = in_io_id; + trx->close_chk_out_io_id = out_io_id; + if (has_traffic || closing || local_tx_event_q == nullptr) { + return (uint64_t)FLAGS_ub_event_queue_timer_interval_us; + } + uint64_t next = cur_interval_us * kCloseCheckBackoffFactor; + const uint64_t max_us = (uint64_t)FLAGS_ub_event_queue_timer_interval_max_us; + if (max_us > 0 && next > max_us) { + next = max_us; + } else if (max_us == 0) { + next = cur_interval_us; + } + return next; +} + RETURN_CODE UBRing::UbrAddCloseTimer() { if (UNLIKELY(_trx == nullptr)) { LOG(ERROR) << "Trx add close timer failed, trx is null."; return UBRING_ERR; } - const uint32_t event_q_timer_interval_ns = - FLAGS_ub_event_queue_timer_interval_us * USEC_TO_NSEC; - itimerspec time_spec = { - .it_interval = {.tv_sec = 0, .tv_nsec = event_q_timer_interval_ns}, - .it_value = {.tv_sec = 0, .tv_nsec = 1} - }; - int timer_fd = TimerStart(&time_spec, UbrTrxCloseCallback, (void*)_trx); - if (UNLIKELY(timer_fd == -1)) { + const uint32_t interval_us = FLAGS_ub_event_queue_timer_interval_us; + _trx->close_chk_in_io_id = ATOMIC_LOAD(_trx->ubr_rx.in_io_id); + _trx->close_chk_out_io_id = ATOMIC_LOAD(_trx->ubr_tx.out_io_id); + RETURN_CODE rc = UbrTimerStart(&_trx->close_timer, 0, interval_us, + UbrTrxCloseCallback, (void*)_trx, + UbrCloseTimerBackoff); + if (UNLIKELY(rc != UBRING_OK)) { LOG(ERROR) << "Start ubr close timer failed, trx local name=" << _trx->local_shm.name; return UBRING_ERR; } - _trx->timer_fd = timer_fd; return UBRING_OK; } @@ -152,7 +333,7 @@ RETURN_CODE UBRing::UbrAddTimer() { } if (UNLIKELY(UbrAddHBTimer() != UBRING_OK)) { - DeleteTimerSafe((uint32_t)_trx->timer_fd); + UbrTimerDelAndWait(&_trx->close_timer); LOG(ERROR) << "Ubr " << _trx->local_shm.name << " add heartbeat timer failed."; return UBRING_ERR; } @@ -171,15 +352,12 @@ void* UBRing::UbrTrxCloseCallback(void* args) { return nullptr; } trx->ubr_rx.trx_state = UBR_STATE_CLOSED; - int fd = (int)trx->local_shm.fd; do { if (ATOMIC_LOAD(trx->close_cnt) == 0) { break; } ATOMIC_SUB(trx->close_cnt, 1); - uint64_t start_time = GetCurNanoSeconds(); - if (local_tx_event_q->flag == UBR_STATE_CONNECTED || ATOMIC_LOAD(trx->close_cnt) == 1) { local_tx_event_q->flag = UBR_STATE_CLOSED; trx->ubr_tx.trx_state = UBR_STATE_CLOSED; @@ -190,7 +368,7 @@ void* UBRing::UbrTrxCloseCallback(void* args) { break; } remote_rx_event_q->flag = UBR_STATE_CLOSED; - RETURN_CODE clear_rc = ClearTrxResource(trx, start_time, UBR_CALL_BACK_CLOSE, 1); + RETURN_CODE clear_rc = ClearTrxResource(trx); if (UNLIKELY(clear_rc != UBRING_OK && clear_rc != UBRING_REENTRY)) { LOG(ERROR) << "Trx close callback failed, " << trx->local_shm.name << " clear trx resource failed."; break; @@ -205,54 +383,56 @@ RETURN_CODE UBRing::UbrAddHBTimer() { return UBRING_ERR; } - itimerspec time_spec = { - .it_interval = {.tv_sec = FLAGS_ub_hb_timer_interval_s, .tv_nsec = 0}, - .it_value = {.tv_sec = 0, .tv_nsec = 1} - }; - int timer_fd = TimerStart(&time_spec, UbrTrxHBCallback, (void*)_trx); - if (UNLIKELY(timer_fd == -1)) { + const uint64_t interval_us = (uint64_t)FLAGS_ub_hb_timer_interval_s * SEC_TO_USEC; + RETURN_CODE rc = UbrTimerStart(&_trx->hb_timer, 0, interval_us, + UbrTrxHBCallback, (void*)_trx); + if (UNLIKELY(rc != UBRING_OK)) { LOG(ERROR) << "Start ubr heartbeat timer failed."; return UBRING_ERR; } - _trx->hb_timer_fd = timer_fd; return UBRING_OK; } -RETURN_CODE UBRing::UbrPassiveClearTrx(UbrTrx *trx, int fd, PASSIVE_DISC_TYPE type) { +RETURN_CODE UBRing::UbrPassiveClearTrx(UbrTrx *trx) { RETURN_CODE passive_close_check_rc = UbrTrxCloseCheck(trx); if (UNLIKELY(passive_close_check_rc != UBRING_OK)) { if (passive_close_check_rc == UBRING_REENTRY) { LOG(INFO) << "Passive close skipped, active close in progress, name=" << trx->local_shm.name; - uint64_t start_time = GetCurNanoSeconds(); - return ClearTrxResource(trx, start_time, UBR_CALL_BACK_CLOSE); + return ClearTrxResource(trx); } return UBRING_ERR; } trx->ubr_tx.trx_state = UBR_STATE_CLOSED; trx->ubr_rx.trx_state = UBR_STATE_CLOSED; - DeleteTimerSafe((uint32_t)trx->timer_fd); - const char *type_name = nullptr; - if (type == UBR_HEARTBEAT) { - DeleteTimer((uint32_t)trx->hb_timer_fd); - type_name = "Trx heartbeat"; - } else if (type == UBR_UB_EVENT) { - DeleteTimerSafe((uint32_t)trx->hb_timer_fd); - type_name = "Ub event callback"; - } - constexpr int64_t kMicrosecondsPerSecond = 1000000LL; - bthread_usleep(FLAGS_ub_flying_io_timeout_s * kMicrosecondsPerSecond); + // Non-blocking: this may run inside the heartbeat callback itself. + UbrTimerDel(&trx->close_timer); + UbrTimerDel(&trx->hb_timer); + // Wait for in-flight IO on a one-shot timer instead of sleeping on the + // timer thread. + return UbrScheduleClearTimer(trx, UbrPassiveClearCallback, UbrDoPassiveClearWork); +} - int rc = ShmLocalFree(&trx->remote_shm); - if (rc != UBRING_OK) { - LOG(ERROR) << type_name << ", delete remote shm failed. ret=" << rc; +void* UBRing::UbrPassiveClearCallback(void* args) { + auto* ctl = (UbrCleanupCtl*)args; + if (UNLIKELY(ctl == nullptr)) { + LOG(ERROR) << "Trx passive clear callback failed, ctl is null."; + return nullptr; } - rc = ShmLocalFree(&trx->local_shm); - if (rc != UBRING_OK) { - LOG(ERROR) << type_name << ", delete local shm failed. ret=" << rc; + int expected = UBR_CLEANUP_PENDING; + if (!ATOMIC_COMPARE_EXCHANGE_STRONG(ctl->state, + expected, UBR_CLEANUP_RUNNING)) { + // Force close owns the cleanup; this fire still holds the + // timer/callback reference inherited from the schedule. + ctl->ReleaseRef(); + return nullptr; } - - UBRingManager::ReleaseUbrTrxFromMgr(trx); - return UBRING_OK; + UbrTrx* trx = ctl->trx; + if (UNLIKELY(UBRingManager::IsUbrTrxSlotUsed(trx->trx_mgr_index, ctl->ubr_id))) { + UbrDoPassiveClearWork(trx, ctl->ubr_id); + } + ATOMIC_STORE(ctl->state, UBR_CLEANUP_DONE); + ctl->ReleaseRef(); // timer/callback reference + return nullptr; } void* UBRing::UbrTrxHBCallback(void* args) { @@ -286,9 +466,9 @@ void* UBRing::UbrTrxHBCallback(void* args) { } int fd = (int)trx->local_shm.fd; - LOG(INFO) << "Hlc heartbeat, start to clear trx resource. hb_timer_fd=" << fd << ", shm_name=" << trx->local_shm.name; - UbrPassiveClearTrx(trx, fd, UBR_HEARTBEAT); - LOG(INFO) << "Hlc heartbeat clear trx resource finish."; + LOG(INFO) << "Ubr heartbeat, start to clear trx resource. shm_fd=" << fd << ", shm_name=" << trx->local_shm.name; + UbrPassiveClearTrx(trx); + LOG(INFO) << "Ubr heartbeat clear trx resource finish."; return nullptr; } @@ -297,40 +477,30 @@ RETURN_CODE UBRing::UbrAddAsynClearTimer(UbrTrx *trx) { LOG(ERROR) << "Trx add close timer failed, trx is null."; return UBRING_ERR; } - - if (trx->clear_timer_fd > 0) { - return UBRING_OK; - } - - itimerspec time_spec = { - .it_interval = {.tv_sec = 0, .tv_nsec = 0}, - .it_value = {.tv_sec = FLAGS_ub_flying_io_timeout_s, .tv_nsec = 0} - }; - - int timer_fd = TimerStart(&time_spec, UbrAsynClearCallback, (void*)trx); - if (UNLIKELY(timer_fd == -1)) { - LOG(ERROR) << "Start ubr close timer failed, trx name=%s.", trx->local_shm.name; - return UBRING_ERR; - } - trx->clear_timer_fd = timer_fd; - return UBRING_OK; + return UbrScheduleClearTimer(trx, UbrAsynClearCallback, UbrDoAsynClearWork); } void *UBRing::UbrAsynClearCallback(void *args) { - auto* trx = (UbrTrx*) args; - if (UNLIKELY(trx == nullptr)) { - LOG(ERROR) << "Trx close, trx is null."; + auto* ctl = (UbrCleanupCtl*) args; + if (UNLIKELY(ctl == nullptr)) { + LOG(ERROR) << "Trx close, ctl is null."; return nullptr; } - - if (UNLIKELY(UbrTrxFreeShm(trx) != UBRING_OK)) { - LOG(ERROR) << "Trx close, wait for local shm " << trx->local_shm.name << " free fail."; + int expected = UBR_CLEANUP_PENDING; + if (!ATOMIC_COMPARE_EXCHANGE_STRONG(ctl->state, + expected, UBR_CLEANUP_RUNNING)) { + // Force close owns the cleanup; this fire still holds the + // timer/callback reference inherited from the schedule. + ctl->ReleaseRef(); + return nullptr; } - - if (UNLIKELY(UBRingManager::ReleaseUbrTrxFromMgr(trx) != UBRING_OK)) { - LOG(ERROR) << "Trx close, release shm " << trx->local_shm.name << " trx failed."; + UbrTrx* trx = ctl->trx; + if (UNLIKELY(UBRingManager::IsUbrTrxSlotUsed(trx->trx_mgr_index, ctl->ubr_id))) { + UbrDoAsynClearWork(trx, ctl->ubr_id); } + ATOMIC_STORE(ctl->state, UBR_CLEANUP_DONE); + ctl->ReleaseRef(); // timer/callback reference return nullptr; } @@ -360,8 +530,8 @@ int UBRing::UbrTrxSend(const void *buf, uint32_t buf_len) uint32_t total_send_len = 0; uint32_t remain_buf_len = buf_len; uint8_t is_last_pkt = 0; - _trx->ubr_tx.out_io_id++; - ((UbrEventQMsg *)_trx->ubr_tx.remote_rx_event_q.addr)->io_id = _trx->ubr_tx.out_io_id; + const uint64_t io_seq = ATOMIC_ADD(_trx->ubr_tx.out_io_id, 1) + 1; + ((UbrEventQMsg *)_trx->ubr_tx.remote_rx_event_q.addr)->io_id = io_seq; while (remain_buf_len > 0) { is_last_pkt = (uint8_t)(remain_buf_len <= UBR_MSG_PAYLOAD_LEN); msg->header[UBR_MSG_FLAG_INDEX] = is_last_pkt ? UBR_MSG_CHUNK_EOF : UBR_MSG_CHUNK_EXIST; @@ -468,8 +638,8 @@ ssize_t UBRing::UbrTrxWritev(const struct iovec *iov, int iovcnt) size_t fulled = 0; uint8_t is_last_pkt = 0; uint8_t cur_pkt_len = 0; - _trx->ubr_tx.out_io_id++; - ((UbrEventQMsg *)_trx->ubr_tx.remote_rx_event_q.addr)->io_id = _trx->ubr_tx.out_io_id; + const uint64_t io_seq = ATOMIC_ADD(_trx->ubr_tx.out_io_id, 1) + 1; + ((UbrEventQMsg *)_trx->ubr_tx.remote_rx_event_q.addr)->io_id = io_seq; while (buf_len > 0) { is_last_pkt = (uint8_t)(buf_len <= UBR_MSG_PAYLOAD_LEN); cur_pkt_len = is_last_pkt ? (uint8_t)buf_len : UBR_MSG_PAYLOAD_LEN; @@ -557,7 +727,7 @@ RETURN_CODE UBRing::IsUbrTrxReadable(uint32_t ep_event) } uint64_t io_id = ((UbrEventQMsg *)_trx->ubr_rx.local_rx_event_q.addr)->io_id; - if ((ep_event & EPOLLET) && io_id == _trx->ubr_rx.in_io_id) { + if ((ep_event & EPOLLET) && io_id == ATOMIC_LOAD(_trx->ubr_rx.in_io_id)) { return MPA_MUXER_NOT_READY; } @@ -572,7 +742,7 @@ RETURN_CODE UBRing::IsUbrTrxReadable(uint32_t ep_event) return MPA_MUXER_NOT_READY; } if (ep_event & EPOLLET) { - _trx->ubr_rx.in_io_id = io_id; + ATOMIC_STORE(_trx->ubr_rx.in_io_id, io_id); } return UBRING_OK; } @@ -828,7 +998,7 @@ int UBRing::UbrAllocateServerShm(SHM* remote_trx_shm, SHM* local_trx_shm) { if (UNLIKELY((UbrServerTrxInit(local_trx_shm, remote_trx_shm)) != UBRING_OK)) { LOG(ERROR) << "Server trx init failed."; UbrTrxFreeShm(_trx); - UBRingManager::ReleaseUbrTrxFromMgr(_trx); + UBRingManager::ReleaseUbrTrxFromMgr(_trx, ATOMIC_LOAD(_trx->ubr_id)); _trx = nullptr; return -1; } @@ -881,7 +1051,8 @@ RETURN_CODE UBRing::UbrMapRemoteShmAddTimer(SHM *local_trx_shm, const char *loca local_name, SERVER_SHM_NAME_SUFFIX); if (UNLIKELY(result < 0)) { - LOG(ERROR) << "Copy server shared memory name failed, local_name=%s, ret=%d.", local_name, result; + LOG(ERROR) << "Copy server shared memory name failed, local_name=" << local_name + << ", ret=" << result; return UBRING_ERR; } UbrSetSleepTask(UBR_TASK_CONNECT_MAP_FRONT); @@ -902,8 +1073,8 @@ RETURN_CODE UBRing::UbrMapRemoteShmAddTimer(SHM *local_trx_shm, const char *loca uint32_t timeout = ((UbrDataStatusQMsg *)(_trx->ubr_tx.local_data_status_q.addr))->timeout; if (HasTimedOut(start_time, timeout) != UBRING_OK) { LOG(ERROR) << "Local shm " << local_trx_shm->name << " wait for connect remote map timeout."; - DeleteTimerSafe((uint32_t)_trx->hb_timer_fd); - DeleteTimerSafe((uint32_t)_trx->timer_fd); + UbrTimerDelAndWait(&_trx->hb_timer); + UbrTimerDelAndWait(&_trx->close_timer); ShmRemoteFree(&_trx->remote_shm); return UBRING_ERR_TIMEOUT; } @@ -934,14 +1105,14 @@ RETURN_CODE UBRing::ApplyAndMapLocalShm(SHM *local_trx_shm, const char *local_na if (rc == SHM_ERR_EXIST || rc == SHM_ERR_NOT_FOUND) { rc = UBR_ERR_ADDR_IN_USE; } - UBRingManager::ReleaseUbrTrxFromMgr(_trx); + UBRingManager::ReleaseUbrTrxFromMgr(_trx, ATOMIC_LOAD(_trx->ubr_id)); return rc; } rc = UbrTrxMapLocalShm(local_trx_shm); if (UNLIKELY(rc != UBRING_OK)) { LOG(ERROR) << "Trx map local shared memory failed, local shm name=" << local_trx_shm->name; ShmLocalFree(local_trx_shm); - UBRingManager::ReleaseUbrTrxFromMgr(_trx); + UBRingManager::ReleaseUbrTrxFromMgr(_trx, ATOMIC_LOAD(_trx->ubr_id)); return rc; } ((UbrDataStatusQMsg *)_trx->ubr_tx.local_data_status_q.addr)->timeout = @@ -951,7 +1122,7 @@ RETURN_CODE UBRing::ApplyAndMapLocalShm(SHM *local_trx_shm, const char *local_na if (rc != UBRING_OK) { LOG(ERROR) << "Get ubring deal msg max cnt, local shm name=" << local_trx_shm->name; ShmLocalFree(local_trx_shm); - UBRingManager::ReleaseUbrTrxFromMgr(_trx); + UBRingManager::ReleaseUbrTrxFromMgr(_trx, ATOMIC_LOAD(_trx->ubr_id)); return rc; } return UBRING_OK; @@ -994,7 +1165,7 @@ RETURN_CODE UBRing::WritevHasEnoughSpace(size_t buf_len) return UBRING_OK; } -RETURN_CODE UBRing::UbrClearResourceCheck(UbrTrx *trx, uint64_t start_time, UbrCloseType close_type) +RETURN_CODE UBRing::UbrClearResourceCheck(UbrTrx *trx) { if (UNLIKELY(trx == nullptr)) { LOG(ERROR) << "Trx close failed, trx is null."; @@ -1002,16 +1173,17 @@ RETURN_CODE UBRing::UbrClearResourceCheck(UbrTrx *trx, uint64_t start_time, UbrC } UbrEventQMsg* local_tx_event_q = (UbrEventQMsg *)trx->ubr_tx.local_tx_event_q.addr; + if (UNLIKELY(local_tx_event_q == nullptr)) { + LOG(ERROR) << "Trx close failed, local_tx_event_q addr is NULL, trx local name=" << trx->local_shm.name; + return UBRING_ERR; + } if (local_tx_event_q->flag == UBR_STATE_CONNECTED) { local_tx_event_q->flag = UBR_STATE_CLOSING; } - if (close_type == UBR_SEND_CLOSE) { - DeleteTimerSafe((uint32_t)trx->timer_fd); - } else { - DeleteTimer((uint32_t)trx->timer_fd); - } - DeleteTimerSafe((uint32_t)trx->hb_timer_fd); + // Non-blocking: may run inside the close callback itself. + UbrTimerDel(&trx->close_timer); + UbrTimerDel(&trx->hb_timer); if (local_tx_event_q->flag == UBR_STATE_CLOSING) { local_tx_event_q->flag = UBR_STATE_CLOSED; @@ -1021,9 +1193,9 @@ RETURN_CODE UBRing::UbrClearResourceCheck(UbrTrx *trx, uint64_t start_time, UbrC return UBRING_OK; } -RETURN_CODE UBRing::ClearTrxResource(UbrTrx *trx, uint64_t start_time, UbrCloseType close_type, int op) +RETURN_CODE UBRing::ClearTrxResource(UbrTrx *trx) { - RETURN_CODE rc = UbrClearResourceCheck(trx, start_time, close_type); + RETURN_CODE rc = UbrClearResourceCheck(trx); if (rc != UBRING_OK) { return rc; } diff --git a/src/brpc/ubshm/ub_ring.h b/src/brpc/ubshm/ub_ring.h index 412c759885..656317447b 100644 --- a/src/brpc/ubshm/ub_ring.h +++ b/src/brpc/ubshm/ub_ring.h @@ -55,12 +55,14 @@ class UBRing : public butil::IReader { static void *UbrTrxHBCallback(void *args); - static RETURN_CODE UbrPassiveClearTrx(UbrTrx *trx, int fd, PASSIVE_DISC_TYPE type); + static RETURN_CODE UbrPassiveClearTrx(UbrTrx *trx); static RETURN_CODE UbrAddAsynClearTimer(UbrTrx *trx); static void *UbrAsynClearCallback(void *args); + static void *UbrPassiveClearCallback(void *args); + int UbrTrxSend(const void *buf, uint32_t buf_len); int UbrTrxRecv(void *buf, uint32_t buf_len); @@ -197,8 +199,8 @@ class UBRing : public butil::IReader { void PreWriteAddr(uint8_t *addr, size_t len); RETURN_CODE WritevHasEnoughSpace(size_t buf_len); RETURN_CODE UbrServerTrxInit(SHM *local_shm, SHM *remote_shm); - static RETURN_CODE UbrClearResourceCheck(UbrTrx *trx, uint64_t start_time, UbrCloseType close_type); - static RETURN_CODE ClearTrxResource(UbrTrx *trx, uint64_t start_time, UbrCloseType close_type, int op=0); + static RETURN_CODE UbrClearResourceCheck(UbrTrx *trx); + static RETURN_CODE ClearTrxResource(UbrTrx *trx); UbrTrx* _trx{nullptr}; }; diff --git a/src/brpc/ubshm/ub_ring_manager.cpp b/src/brpc/ubshm/ub_ring_manager.cpp index 64f2434eba..6abb485e8a 100644 --- a/src/brpc/ubshm/ub_ring_manager.cpp +++ b/src/brpc/ubshm/ub_ring_manager.cpp @@ -22,6 +22,11 @@ namespace brpc { namespace ubring { + +// A UbrCleanupCtl reference count of 1 means only the manager anchor is +// left, i.e. no cleanup callback is in flight for it. +static constexpr int kAnchoredRefOnly = 1; + DEFINE_int32(ubr_max_managed_num, 1024, "maximum number of managed ubring"); DEFINE_int32(tail_update_after_read, 8, "Position of the tail update after the read"); @@ -54,6 +59,8 @@ RETURN_CODE UBRingManager::UbrMgrDefault() g_ubr_mgr.trx_cap = FLAGS_ubr_max_managed_num; g_ubr_mgr.trx_mgr_unit_status = nullptr; g_ubr_mgr.trx_mgr = nullptr; + g_ubr_mgr.trx_mgr_unit_id = nullptr; + g_ubr_mgr.trx_mgr_unit_ctl = nullptr; return UBRING_OK; } @@ -68,8 +75,14 @@ RETURN_CODE UBRingManager::UbrMgrInit() { g_ubr_mgr.trx_mgr = (UbrTrx *)malloc(trx_mgr_size); size_t trx_mgr_status_size = g_ubr_mgr.trx_cap * sizeof(UbrMgrUnitStatus); g_ubr_mgr.trx_mgr_unit_status = (UbrMgrUnitStatus *)malloc(trx_mgr_status_size); + size_t trx_mgr_id_size = g_ubr_mgr.trx_cap * sizeof(uint64_t); + g_ubr_mgr.trx_mgr_unit_id = (uint64_t *)malloc(trx_mgr_id_size); + size_t trx_mgr_ctl_size = g_ubr_mgr.trx_cap * sizeof(UbrCleanupCtl *); + g_ubr_mgr.trx_mgr_unit_ctl = (UbrCleanupCtl **)malloc(trx_mgr_ctl_size); if (UNLIKELY(g_ubr_mgr.trx_mgr == nullptr || - g_ubr_mgr.trx_mgr_unit_status == nullptr)) { + g_ubr_mgr.trx_mgr_unit_status == nullptr || + g_ubr_mgr.trx_mgr_unit_id == nullptr || + g_ubr_mgr.trx_mgr_unit_ctl == nullptr)) { LOG(ERROR) << "Ubr manager memory allocation failed."; UbrMgrFini(); return UBRING_ERR; @@ -77,15 +90,57 @@ RETURN_CODE UBRingManager::UbrMgrInit() { memset(g_ubr_mgr.trx_mgr, 0, trx_mgr_size); memset(g_ubr_mgr.trx_mgr_unit_status, UBR_MGR_UNIT_FREE, trx_mgr_status_size); + memset(g_ubr_mgr.trx_mgr_unit_id, 0, trx_mgr_id_size); + memset(g_ubr_mgr.trx_mgr_unit_ctl, 0, trx_mgr_ctl_size); LinkInfoInit(); return UBRING_OK; } void UBRingManager::UbrMgrFini() { + // Cancel the pending delayed cleanups and wait for the in-flight ones + // (each holds one extra reference) to finish, before the pool memory + // they touch is freed. A ctl whose timer is still starting can only be + // cancelled in a later round, hence the retry-to-stability loop. + bool busy = true; + while (busy) { + busy = false; + { + LOCK_GUARD(g_ubr_trx_mgr_mtx); + if (g_ubr_mgr.trx_mgr_unit_ctl != nullptr) { + for (uint32_t i = 0; i < g_ubr_mgr.trx_cap; ++i) { + UbrCleanupCtl* ctl = g_ubr_mgr.trx_mgr_unit_ctl[i]; + if (ctl == nullptr) { + continue; + } + if (UbrTimerDel(&ctl->timer) == 0) { + ctl->ReleaseRef(); // timer/callback reference + } + if (ctl->ref.load() > kAnchoredRefOnly) { + busy = true; + } + } + } + } + if (busy) { + LOG_EVERY_SECOND(INFO) << "UbrMgrFini waits for in-flight cleanups."; + usleep(1000); + } + } { LOCK_GUARD(g_ubr_trx_mgr_mtx); + if (g_ubr_mgr.trx_mgr_unit_ctl != nullptr) { + for (uint32_t i = 0; i < g_ubr_mgr.trx_cap; ++i) { + UbrCleanupCtl* ctl = g_ubr_mgr.trx_mgr_unit_ctl[i]; + if (ctl != nullptr) { + g_ubr_mgr.trx_mgr_unit_ctl[i] = nullptr; + ctl->ReleaseRef(); // manager reference + } + } + } FREE_PTR(g_ubr_mgr.trx_mgr); FREE_PTR(g_ubr_mgr.trx_mgr_unit_status); + FREE_PTR(g_ubr_mgr.trx_mgr_unit_id); + FREE_PTR(g_ubr_mgr.trx_mgr_unit_ctl); } { LOCK_GUARD(g_ubr_listener_mgr_mtx); @@ -115,10 +170,25 @@ RETURN_CODE UBRingManager::AcquireUbrTrxFromMgr(UbrTrx **trx) { for (uint32_t i = 0; i < g_ubr_mgr.trx_cap; ++i) { if (g_ubr_mgr.trx_mgr_unit_status[i] == UBR_MGR_UNIT_FREE) { memset(&g_ubr_mgr.trx_mgr[i], 0, sizeof(UbrTrx)); + // The explicit re-initialization after memset is deliberate: it + // documents the per-acquisition invariants of these fields. + g_ubr_mgr.trx_mgr[i].close_timer = nullptr; + g_ubr_mgr.trx_mgr[i].hb_timer = nullptr; + g_ubr_mgr.trx_mgr[i].cleanup_ctl = nullptr; + // Retire the previous acquisition's cleanup control object. + UbrCleanupCtl* old_ctl = g_ubr_mgr.trx_mgr_unit_ctl[i]; + g_ubr_mgr.trx_mgr_unit_ctl[i] = nullptr; + if (old_ctl != nullptr) { + if (UbrTimerDel(&old_ctl->timer) == 0) { + old_ctl->ReleaseRef(); // timer/callback reference + } + old_ctl->ReleaseRef(); // manager anchor + } g_ubr_mgr.trx_mgr_unit_status[i] = UBR_MGR_UNIT_USED; *trx = &g_ubr_mgr.trx_mgr[i]; (*trx)->trx_mgr_index = i; - (*trx)->ubr_id = g_ubr_trx_num; + ATOMIC_STORE((*trx)->ubr_id, g_ubr_trx_num); + g_ubr_mgr.trx_mgr_unit_id[i] = g_ubr_trx_num; (*trx)->close_state = UBR_CLOSE_FIRST; (*trx)->close_cnt = MAX_CLOSE_COUNT; ++g_ubr_mgr.trx_num; @@ -130,17 +200,12 @@ RETURN_CODE UBRingManager::AcquireUbrTrxFromMgr(UbrTrx **trx) { return UBRING_ERR; } -RETURN_CODE UBRingManager::ReleaseUbrTrxFromMgr(UbrTrx *trx) { +RETURN_CODE UBRingManager::ReleaseUbrTrxFromMgr(UbrTrx *trx, + uint64_t expect_ubr_id) { if (UNLIKELY(trx == nullptr)) { LOG(ERROR) << "Release trx failed, trx is null."; return UBRING_ERR; } - - trx->local_shm.addr = nullptr; - trx->ubr_tx.local_tx_event_q.addr = nullptr; - trx->ubr_tx.local_data_status_q.addr = nullptr; - trx->ubr_rx.local_rx_event_q.addr = nullptr; - trx->ubr_rx.remote_data_status_q.addr = nullptr; if (UNLIKELY(g_ubr_mgr.trx_mgr == nullptr)) { LOG(ERROR) << "Release trx failed, trx_mgr is null."; return UBRING_ERR; @@ -153,16 +218,82 @@ RETURN_CODE UBRingManager::ReleaseUbrTrxFromMgr(UbrTrx *trx) { return UBRING_OK; } + if (UNLIKELY(g_ubr_mgr.trx_mgr_unit_id[idx] != expect_ubr_id)) { + // The slot was released and acquired again meanwhile; the stale + // caller must not touch the new occupant. + LOG(WARNING) << "Release stale trx refused, name=" << trx->local_shm.name; + return UBRING_OK; + } + if (g_ubr_mgr.trx_num == 0) { LOG(ERROR) << "Release trx failed, trx number is 0."; return UBRING_ERR; } + // Mutate the trx only after the generation check passed. + trx->local_shm.addr = nullptr; + trx->ubr_tx.local_tx_event_q.addr = nullptr; + trx->ubr_tx.local_data_status_q.addr = nullptr; + trx->ubr_rx.local_rx_event_q.addr = nullptr; + trx->ubr_rx.remote_data_status_q.addr = nullptr; g_ubr_mgr.trx_mgr_unit_status[idx] = UBR_MGR_UNIT_FREE; --g_ubr_mgr.trx_num; return UBRING_OK; } +UbrCleanupCtl* UBRingManager::SnapshotUnitCleanupCtl(uint32_t idx) { + LOCK_GUARD(g_ubr_trx_mgr_mtx); + if (UNLIKELY(g_ubr_mgr.trx_mgr_unit_ctl == nullptr || idx >= g_ubr_mgr.trx_cap)) { + return nullptr; + } + UbrCleanupCtl* ctl = g_ubr_mgr.trx_mgr_unit_ctl[idx]; + if (ctl != nullptr) { + ctl->ref.fetch_add(1); // snapshot reference + } + return ctl; +} + +bool UBRingManager::IsUbrTrxSlotUsed(uint32_t idx, uint64_t expect_ubr_id) { + LOCK_GUARD(g_ubr_trx_mgr_mtx); + if (UNLIKELY(g_ubr_mgr.trx_mgr_unit_id == nullptr || + g_ubr_mgr.trx_mgr_unit_status == nullptr || + idx >= g_ubr_mgr.trx_cap)) { + return false; + } + return g_ubr_mgr.trx_mgr_unit_status[idx] == UBR_MGR_UNIT_USED && + g_ubr_mgr.trx_mgr_unit_id[idx] == expect_ubr_id; +} + +bool UBRingManager::TryPublishUnitCleanupCtl(uint32_t idx, + uint64_t expect_ubr_id, + UbrCleanupCtl *ctl) { + LOCK_GUARD(g_ubr_trx_mgr_mtx); + if (UNLIKELY(g_ubr_mgr.trx_mgr_unit_ctl == nullptr || + g_ubr_mgr.trx_mgr_unit_status == nullptr || + g_ubr_mgr.trx_mgr_unit_id == nullptr || + idx >= g_ubr_mgr.trx_cap || + g_ubr_mgr.trx_mgr_unit_status[idx] != UBR_MGR_UNIT_USED || + g_ubr_mgr.trx_mgr_unit_id[idx] != expect_ubr_id || + g_ubr_mgr.trx_mgr_unit_ctl[idx] != nullptr)) { + return false; // released / reused / already anchored + } + ctl->ref.fetch_add(1); // manager anchor reference + g_ubr_mgr.trx_mgr_unit_ctl[idx] = ctl; + return true; +} + +bool UBRingManager::DetachUnitCleanupCtl(uint32_t idx, UbrCleanupCtl *ctl) { + LOCK_GUARD(g_ubr_trx_mgr_mtx); + if (UNLIKELY(g_ubr_mgr.trx_mgr_unit_ctl == nullptr || + idx >= g_ubr_mgr.trx_cap || + g_ubr_mgr.trx_mgr_unit_ctl[idx] != ctl)) { + return false; + } + g_ubr_mgr.trx_mgr_unit_ctl[idx] = nullptr; + ctl->ReleaseRef(); // manager reference + return true; +} + void UBRingManager::LinkInfoInit(void) { size_t link_info_mgr_size = FLAGS_ubr_max_managed_num * sizeof(UbrLinkInfo); @@ -255,7 +386,7 @@ int32_t UBRingManager::UbEventCallback(const char *shm_name) ++g_ub_event_cnt; int fd = (int)g_ubr_mgr.trx_mgr[i].local_shm.fd; LOG(WARNING) << "Ub event callback, the fd of the faulty link is " << fd; - return UBRing::UbrPassiveClearTrx(&g_ubr_mgr.trx_mgr[i], fd, UBR_UB_EVENT); + return UBRing::UbrPassiveClearTrx(&g_ubr_mgr.trx_mgr[i]); } } return UBRING_ERR; diff --git a/src/brpc/ubshm/ub_ring_manager.h b/src/brpc/ubshm/ub_ring_manager.h index e2afa324c7..131a749ef7 100644 --- a/src/brpc/ubshm/ub_ring_manager.h +++ b/src/brpc/ubshm/ub_ring_manager.h @@ -34,6 +34,8 @@ typedef struct TagUbrMgr { uint32_t trx_cap; UbrTrx *trx_mgr; UbrMgrUnitStatus *trx_mgr_unit_status; + uint64_t *trx_mgr_unit_id; + UbrCleanupCtl **trx_mgr_unit_ctl; } UbrMgr; typedef struct TagUbrLinkInfo { @@ -63,7 +65,34 @@ class UBRingManager { static RETURN_CODE AcquireUbrTrxFromMgr(UbrTrx **trx); - static RETURN_CODE ReleaseUbrTrxFromMgr(UbrTrx *trx); + // Release the pool slot of `trx'. `expect_ubr_id' must be snapshotted + // from trx->ubr_id before the caller started releasing the trx: a + // release racing a reuse of the same slot is refused. + static RETURN_CODE ReleaseUbrTrxFromMgr(UbrTrx *trx, + uint64_t expect_ubr_id); + + // Snapshot the cleanup control object of a pool slot. The caller + // receives a reference and must ReleaseRef it on every exit path. + static UbrCleanupCtl* SnapshotUnitCleanupCtl(uint32_t idx); + + // Under the manager lock, confirm the pool slot is still used by the + // given generation (not released or reused meanwhile). + static bool IsUbrTrxSlotUsed(uint32_t idx, uint64_t expect_ubr_id); + + // Atomically (under the manager lock) anchor `ctl' to the pool slot, + // but only while the slot is still used by the given generation and + // carries no other cleanup control object. On success the ctl gains + // the manager anchor reference (released when the anchor is detached + // or the slot is retired); returns false -- leaving the slot and the + // reference counts untouched -- when the trx was released, reused, or + // a cleanup is already anchored. + static bool TryPublishUnitCleanupCtl(uint32_t idx, uint64_t expect_ubr_id, + UbrCleanupCtl *ctl); + + // Detach `ctl' from the pool slot if it is still anchored there. + // Returns true when the detach happened (the manager reference was + // released). + static bool DetachUnitCleanupCtl(uint32_t idx, UbrCleanupCtl *ctl); static void LinkInfoInit(void); static void LinkInfoFini(void); diff --git a/src/brpc/ubshm/ubr_trx.h b/src/brpc/ubshm/ubr_trx.h index b4e89645ff..6a947d2d78 100644 --- a/src/brpc/ubshm/ubr_trx.h +++ b/src/brpc/ubshm/ubr_trx.h @@ -24,6 +24,7 @@ #include "brpc/ubshm/common/common.h" #include "brpc/ubshm/common/thread_lock.h" #include "brpc/ubshm/ubr_msg.h" +#include "brpc/ubshm/timer/timer_mgr.h" /* +----------------------------------------------------------------------------+ │ UbrTrx shm │ @@ -57,17 +58,22 @@ typedef enum { UBR_STATE_CLOSED } EventQState; -typedef enum { - UBR_SEND_CLOSE, - UBR_CALL_BACK_CLOSE -} UbrCloseType; - typedef enum { UBR_CLOSE_FIRST, UBR_CLOSE_SECOND, UBR_CLOSE_END } UbrCloseCount; +// Ownership of the final delayed cleanup of a trx. Exactly one side (the +// delayed-clear callback or a force close) may run it. The state lives in +// a per-acquisition heap control object (UbrCleanupCtl) anchored by the +// trx manager, so it survives the pool slot reuse that wipes UbrTrx. +typedef enum { + UBR_CLEANUP_PENDING = 1, // cleanup timer scheduled + UBR_CLEANUP_RUNNING = 2, // cleanup owned, in progress + UBR_CLEANUP_DONE = 3 // cleanup finished +} UbrCleanupState; + typedef enum { UDP_TRX, TCP_TRX, @@ -104,11 +110,11 @@ typedef struct TagUbrTx { UbrAddrInfo remote_rx_event_q; UbrAddrInfo local_data_status_q; UbrAddrInfo local_tx_event_q; - uint64_t out_io_id; + AtomicUintFast64 out_io_id; uint32_t write_pos; uint32_t capacity; UbrMsgFormat local_msg_space; - uint32_t hb_retry_cnt; + int32_t hb_retry_cnt; uint32_t ep_last_cap; volatile EventQState trx_state; } UbrTx; @@ -118,7 +124,7 @@ typedef struct TagUbrRx { UbrAddrInfo local_rx_event_q; UbrAddrInfo remote_data_status_q; UbrAddrInfo remote_tx_event_q; - uint64_t in_io_id; + AtomicUintFast64 in_io_id; uint32_t read_pos; uint32_t capacity; uint32_t deal_msg_num; @@ -127,17 +133,47 @@ typedef struct TagUbrRx { volatile EventQState trx_state; } UbrRx; +struct TagUbrTrx; + +// Control object of one delayed cleanup, created when the cleanup is +// scheduled and destroyed when the owning pool slot is acquired again. +// It is anchored by the trx manager (not by the reusable UbrTrx memory), +// so the ownership claim and the completion signal stay valid across the +// release/reuse of the trx that owns the cleanup. +struct UbrCleanupCtl { + TagUbrTrx* trx; // immutable + uint64_t ubr_id; // immutable generation + AtomicInt state; // UbrCleanupState + UbrTimerId timer; // delayed clear timer + // References: timer/callback (held from a successful schedule until + // the callback fully returned, or released by whoever cancels the + // timer before it fires) + starter (until the schedule call + // completes) + manager anchor (taken when the ctl is anchored to the + // pool slot, released when the anchor is detached or retired). + AtomicInt ref; + + void ReleaseRef() { + if (ref.fetch_sub(1) == 1) { + delete this; + } + } +}; + typedef struct TagUbrTrx { UbrTx ubr_tx; UbrRx ubr_rx; - uint64_t ubr_id; + AtomicUintFast64 ubr_id; uint32_t trx_mgr_index; UbrTrxType type; SHM local_shm; SHM remote_shm; - int timer_fd; - int hb_timer_fd; - int clear_timer_fd; + UbrTimerId close_timer; + UbrTimerId hb_timer; + UbrCleanupCtl* cleanup_ctl; + // Last io ids seen by the close-check timer, used to reset its + // back-off polling interval when the link has traffic. + uint64_t close_chk_in_io_id; + uint64_t close_chk_out_io_id; AtomicInt close_cnt; AtomicInt close_state; } UbrTrx; @@ -152,11 +188,7 @@ typedef struct TagUbrLinkLock { FileLock* file_lock; } UbrLinkLock; -typedef enum { - UBR_UB_EVENT, - UBR_HEARTBEAT, -}PASSIVE_DISC_TYPE; - } } -#endif //BRPC_UBR_TRX_H \ No newline at end of file +#endif //BRPC_UBR_TRX_H + diff --git a/test/brpc_ubring_unittest.cpp b/test/brpc_ubring_unittest.cpp index 53bb0a9bd4..30457e6332 100644 --- a/test/brpc_ubring_unittest.cpp +++ b/test/brpc_ubring_unittest.cpp @@ -35,6 +35,7 @@ DECLARE_int32(ub_disconnect_timeout_s); DECLARE_int32(ub_connect_timeout_s); DECLARE_int32(ub_hb_timer_interval_s); DECLARE_int32(ub_event_queue_timer_interval_us); +DECLARE_int32(ub_event_queue_timer_interval_max_us); DECLARE_int32(ub_flying_io_timeout_s); extern bool g_skip_ub_init; @@ -155,6 +156,7 @@ TEST(UBRingConfigurationTest, time_flags_include_units_and_expected_defaults) { {"ub_connect_timeout_s", "_s", "seconds", "1"}, {"ub_hb_timer_interval_s", "_s", "seconds", "5"}, {"ub_event_queue_timer_interval_us", "_us", "microseconds", "100"}, + {"ub_event_queue_timer_interval_max_us", "_us", "microseconds", "10000"}, {"ub_flying_io_timeout_s", "_s", "seconds", "5"}, }; @@ -174,6 +176,7 @@ TEST(UBRingConfigurationTest, time_flags_include_units_and_expected_defaults) { EXPECT_EQ(1, brpc::ubring::FLAGS_ub_connect_timeout_s); EXPECT_EQ(5, brpc::ubring::FLAGS_ub_hb_timer_interval_s); EXPECT_EQ(100, brpc::ubring::FLAGS_ub_event_queue_timer_interval_us); + EXPECT_EQ(10000, brpc::ubring::FLAGS_ub_event_queue_timer_interval_max_us); EXPECT_EQ(5, brpc::ubring::FLAGS_ub_flying_io_timeout_s); EXPECT_EQ(100U * USEC_TO_NSEC, static_cast( From cdb0f9586d653136838ce08c7299ed33e1dc09ee Mon Sep 17 00:00:00 2001 From: Sherlock0203 <1297399478@qq.com> Date: Tue, 1 Sep 2026 11:50:51 +0800 Subject: [PATCH 2/9] =?UTF-8?q?[fix]=E6=B3=A8=E9=87=8A=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/brpc/ubshm/timer/timer_mgr.cpp | 6 ++++-- src/brpc/ubshm/timer/timer_mgr.h | 24 ++++++++++++++---------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/brpc/ubshm/timer/timer_mgr.cpp b/src/brpc/ubshm/timer/timer_mgr.cpp index c803716a94..d4d4e6d5de 100644 --- a/src/brpc/ubshm/timer/timer_mgr.cpp +++ b/src/brpc/ubshm/timer/timer_mgr.cpp @@ -236,8 +236,10 @@ int UbrTimerDel(UbrTimerId* slot) { } // ==1: dispatched, OnFire (owned==false) // releases it ReleaseRef(task); // owner - return 0; // the callback is guaranteed never to run: the caller - // consumes the timer/callback reference + return 0; // This call won the slot competition. For a one-shot timer, + // the callback will not run. For a periodic timer, future + // rearming is stopped, but an already dispatched or running + // callback may still complete. } void UbrTimerDelAndWait(UbrTimerId* slot) { diff --git a/src/brpc/ubshm/timer/timer_mgr.h b/src/brpc/ubshm/timer/timer_mgr.h index 74e5083ce7..f4d244eceb 100644 --- a/src/brpc/ubshm/timer/timer_mgr.h +++ b/src/brpc/ubshm/timer/timer_mgr.h @@ -45,16 +45,20 @@ RETURN_CODE UbrTimerStart(UbrTimerId* slot, uint64_t delay_us, void* arg, UbrTimerBackoffFn backoff = nullptr); -// Non-blocking delete, safe from inside the timer callback itself. Does -// not wait for a running callback and does not protect `arg' on its own. -// Returns 0 when the call won the slot competition: a one-shot callback -// is guaranteed never to run, and the caller consumes any per-task -// resources it tracks for this timer (ownership of them transfers to the -// caller); for a periodic timer an already-started callback is not -// interrupted. Returns 1 when the callback has been dispatched (it -// consumes those resources itself on every exit) or its fate is still -// being settled by the scheduler -- the caller must not consume anything -// then. +// Non-blocking delete, safe to call from inside the timer callback itself. +// This function does not wait for an already running callback and does not +// protect resources reachable from `arg` on its own. +// +// Returns 0 when this call wins the handle-slot competition. +// - For a one-shot timer, the callback will not run. +// - For a periodic timer, future rearming is stopped, but an already +// dispatched or running callback may still execute once more. Callers +// must not reclaim resources reachable from `arg` based on this return +// alone; use UbrTimerDelAndWait when teardown needs to wait for callbacks. +// +// Returns 1 when this caller did not acquire timer ownership. The callback, +// another deleter, or the scheduling path is responsible for settling the +// timer resources, so this caller must not reclaim them. int UbrTimerDel(UbrTimerId* slot); // Delete and wait until a possibly running callback finished, so the From 7e0dd01df3334c06fe614999939fb2b101d632df Mon Sep 17 00:00:00 2001 From: Sherlock0203 <1297399478@qq.com> Date: Thu, 24 Sep 2026 17:16:33 +0800 Subject: [PATCH 3/9] style(ubshm): use BAIDU macros, BAIDU_SCOPED_LOCK and butil::atomic (#57) Make the whole ubring shared-memory module follow the current brpc conventions, addressing review feedback: - Replace the module-local LIKELY/UNLIKELY aliases with BAIDU_LIKELY/ BAIDU_UNLIKELY and drop the aliases from common.h. - Use BAIDU_SCOPED_LOCK instead of the local GNU cleanup-attribute LOCK_GUARD. The pthread_mutex_t members keep their type, allocation and lifetime, so no static-init or placement-new change is needed. - Use butil::atomic instead of std::atomic. The timer handle slots and the trx cleanup slot are now real butil::atomic objects, so plain storage is never reinterpreted as an atomic: the timer API takes butil::atomic* and UbrTrx/UbrCleanupCtl hold atomic handles. UbrTrx slots are value-initialized with placement new so those atomics are constructed, replacing the per-acquisition memset. - Drop the two per-node INFO logs on the shm cleanup retry path in UbsShmCallback; DeleteShmToList still records each drained node. No behavioral change: the removed aliases expanded 1:1, BAIDU_SCOPED_LOCK locks the same pthread_mutex_t, butil::atomic is layout-identical to std::atomic, and placement-new value-initialization zeroes exactly what memset zeroed while also constructing the atomic members. --- src/brpc/ubshm/common/common.h | 13 +- src/brpc/ubshm/common/thread_lock.h | 2 +- src/brpc/ubshm/shm/shm_mgr.cpp | 20 ++-- src/brpc/ubshm/shm/shm_ubs.cpp | 29 ++--- src/brpc/ubshm/timer/timer_mgr.cpp | 46 ++++---- src/brpc/ubshm/timer/timer_mgr.h | 10 +- src/brpc/ubshm/ub_endpoint.cpp | 8 +- src/brpc/ubshm/ub_endpoint.h | 2 +- src/brpc/ubshm/ub_ring.cpp | 177 +++++++++++++--------------- src/brpc/ubshm/ub_ring.h | 24 ++-- src/brpc/ubshm/ub_ring_manager.cpp | 61 +++++----- src/brpc/ubshm/ubr_trx.h | 8 +- 12 files changed, 194 insertions(+), 206 deletions(-) diff --git a/src/brpc/ubshm/common/common.h b/src/brpc/ubshm/common/common.h index 504da07c28..56d2cb8706 100644 --- a/src/brpc/ubshm/common/common.h +++ b/src/brpc/ubshm/common/common.h @@ -24,9 +24,6 @@ #include "butil/compiler_specific.h" #include "butil/logging.h" -#define LIKELY(x) BAIDU_LIKELY(x) -#define UNLIKELY(x) BAIDU_UNLIKELY(x) - #ifndef UNREFERENCE_PARAM #define UNREFERENCE_PARAM(x) ((void)(x)) #endif @@ -42,11 +39,11 @@ #endif #ifdef __cplusplus -#include -using AtomicInt = std::atomic; -using AtomicBool = std::atomic; -using AtomicUintFast64 = std::atomic; -using AtomicUintFast8 = std::atomic; +#include "butil/atomicops.h" +using AtomicInt = butil::atomic; +using AtomicBool = butil::atomic; +using AtomicUintFast64 = butil::atomic; +using AtomicUintFast8 = butil::atomic; #define ATOMIC_INIT(var, value) var.store(value) #define ATOMIC_STORE(var, value) var.store(value) #define ATOMIC_LOAD(var) var.load() diff --git a/src/brpc/ubshm/common/thread_lock.h b/src/brpc/ubshm/common/thread_lock.h index 0233955168..bba5f80702 100644 --- a/src/brpc/ubshm/common/thread_lock.h +++ b/src/brpc/ubshm/common/thread_lock.h @@ -30,7 +30,7 @@ extern "C" { static inline void UnlockMutex(pthread_mutex_t **mtx) { - if (LIKELY(mtx != nullptr && *mtx != nullptr)) { + if (BAIDU_LIKELY(mtx != nullptr && *mtx != nullptr)) { pthread_mutex_unlock(*mtx); } else { LOG(ERROR) << "Invalid input for mtx."; diff --git a/src/brpc/ubshm/shm/shm_mgr.cpp b/src/brpc/ubshm/shm/shm_mgr.cpp index ea01113e54..3dc434756b 100644 --- a/src/brpc/ubshm/shm/shm_mgr.cpp +++ b/src/brpc/ubshm/shm/shm_mgr.cpp @@ -57,7 +57,7 @@ static bool CheckInputShmParam(SHM *shm) { } RETURN_CODE ShmMgrInit(void) { - if (UNLIKELY(FLAGS_ub_shm_type >= (int32_t)SHM_TYPE_UNSUPPORT || FLAGS_ub_shm_type <= (int32_t)SHM_TYPE_UB)) { + if (BAIDU_UNLIKELY(FLAGS_ub_shm_type >= (int32_t)SHM_TYPE_UNSUPPORT || FLAGS_ub_shm_type <= (int32_t)SHM_TYPE_UB)) { LOG(ERROR) << "Shm type config=" << FLAGS_ub_shm_type << " is not supported."; return UBRING_ERR; } @@ -88,7 +88,7 @@ void SetShmType(SHM_TYPE type) { } RETURN_CODE ShmLocalMalloc(SHM *shm) { - if (UNLIKELY(!CheckInputShmParam(shm))) { + if (BAIDU_UNLIKELY(!CheckInputShmParam(shm))) { LOG(ERROR) << "Input param shm is invalid."; return SHM_ERR_INPUT_INVALID; } @@ -110,11 +110,11 @@ RETURN_CODE ShmLocalMalloc(SHM *shm) { RETURN_CODE ShmLocalCalloc(SHM *shm) { RETURN_CODE rc = ShmLocalMalloc(shm); - if (UNLIKELY(rc != UBRING_OK)) { + if (BAIDU_UNLIKELY(rc != UBRING_OK)) { LOG(ERROR) << "Failed to alloc local shm."; return rc; } - if (UNLIKELY(shm->addr == nullptr)) { + if (BAIDU_UNLIKELY(shm->addr == nullptr)) { LOG(ERROR) << "Local shm=" << shm->name << " allocated with NULL address."; ShmFree(shm); return SHM_ERR; @@ -124,7 +124,7 @@ RETURN_CODE ShmLocalCalloc(SHM *shm) { } RETURN_CODE ShmLocalFree(SHM *shm) { - if (UNLIKELY(!CheckInputShmParam(shm))) { + if (BAIDU_UNLIKELY(!CheckInputShmParam(shm))) { LOG(ERROR) << "Input param shm is invalid."; return SHM_ERR_INPUT_INVALID; } @@ -145,7 +145,7 @@ RETURN_CODE ShmLocalFree(SHM *shm) { } RETURN_CODE ShmRemoteMalloc(SHM *shm) { - if (UNLIKELY(!CheckInputShmParam(shm))) { + if (BAIDU_UNLIKELY(!CheckInputShmParam(shm))) { LOG(ERROR) << "Input param shm is invalid."; return SHM_ERR_INPUT_INVALID; } @@ -166,7 +166,7 @@ RETURN_CODE ShmRemoteMalloc(SHM *shm) { } RETURN_CODE ShmRemoteFree(SHM *shm) { - if (UNLIKELY(!CheckInputShmParam(shm))) { + if (BAIDU_UNLIKELY(!CheckInputShmParam(shm))) { LOG(ERROR) << "Input param shm is invalid."; return SHM_ERR_INPUT_INVALID; } @@ -187,7 +187,7 @@ RETURN_CODE ShmRemoteFree(SHM *shm) { } RETURN_CODE ShmLocalMmap(SHM *shm, int prot) { - if (UNLIKELY(!CheckInputShmParam(shm))) { + if (BAIDU_UNLIKELY(!CheckInputShmParam(shm))) { LOG(ERROR) << "Input param shm is invalid."; return SHM_ERR_INPUT_INVALID; } @@ -208,7 +208,7 @@ RETURN_CODE ShmLocalMmap(SHM *shm, int prot) { } RETURN_CODE ShmMunmap(SHM *shm) { - if (UNLIKELY(!CheckInputShmParam(shm))) { + if (BAIDU_UNLIKELY(!CheckInputShmParam(shm))) { LOG(ERROR) << "Input param shm is invalid."; return SHM_ERR_INPUT_INVALID; } @@ -229,7 +229,7 @@ RETURN_CODE ShmMunmap(SHM *shm) { } RETURN_CODE ShmFree(SHM *shm) { - if (UNLIKELY(!CheckInputShmParam(shm))) { + if (BAIDU_UNLIKELY(!CheckInputShmParam(shm))) { LOG(ERROR) << "Input param shm is invalid."; return SHM_ERR_INPUT_INVALID; } diff --git a/src/brpc/ubshm/shm/shm_ubs.cpp b/src/brpc/ubshm/shm/shm_ubs.cpp index a4f27ee1d8..89f6702a6b 100644 --- a/src/brpc/ubshm/shm/shm_ubs.cpp +++ b/src/brpc/ubshm/shm/shm_ubs.cpp @@ -22,8 +22,8 @@ #include #include #include +#include "butil/scoped_lock.h" #include "brpc/ubshm/timer/timer_mgr.h" -#include "brpc/ubshm/common/thread_lock.h" #include "brpc/ubshm/common/common.h" #include "brpc/ubshm/shm/shm_def.h" #include "brpc/ubshm/ub_ring_manager.h" @@ -47,7 +47,7 @@ DEFINE_int32(ub_flying_io_timeout_s, 5, "Time in seconds to wait for stopping data sending and receiving " "when the link is disconnected."); char g_region_name[MAX_REGION_NAME_DESC_LENGTH] = {0}; -UbrTimerId g_shm_timer_id = nullptr; +butil::atomic g_shm_timer_id(nullptr); ShmList *g_shm_list = nullptr; static RETURN_CODE UbsShmInterfacesLoad(void); char hostname[MAX_HOST_NAME_DESC_LENGTH]; @@ -351,12 +351,12 @@ RETURN_CODE UbsShmInit(void) return UBRING_ERR; } - if (UNLIKELY(ubsmem_local_nid_query(&FLAGS_node_location) != UBSM_OK)) { + if (BAIDU_UNLIKELY(ubsmem_local_nid_query(&FLAGS_node_location) != UBSM_OK)) { LOG(ERROR) << "Get local nid failed."; return UBRING_ERR; } - if (UNLIKELY(ubsmem_shmem_faults_register(brpc::ubring::UBRingManager::UbEventCallback) != UBSM_OK)) { + if (BAIDU_UNLIKELY(ubsmem_shmem_faults_register(brpc::ubring::UBRingManager::UbEventCallback) != UBSM_OK)) { LOG(ERROR) << "Failed to register the ub event callback function."; return UBRING_ERR; } @@ -378,7 +378,7 @@ RETURN_CODE UbsShmInit(void) RETURN_CODE UbsShmFini(void) { // Stop the cleanup timer before finalizing the SDK it calls into. - if (UNLIKELY(DestroyShmTimer(g_shm_list) != UBRING_OK)) { + if (BAIDU_UNLIKELY(DestroyShmTimer(g_shm_list) != UBRING_OK)) { LOG(ERROR) << "Ubs shm list finalize failed."; return UBRING_ERR; } @@ -414,7 +414,7 @@ static void DeleteShmToList(ShmList* shm_list) void *UbsShmCallback(void* args) { ShmList *shm_list = (ShmList*)args; - if (UNLIKELY(shm_list == nullptr)) { + if (BAIDU_UNLIKELY(shm_list == nullptr)) { LOG(ERROR) << "Shm list is null."; return nullptr; } @@ -423,7 +423,7 @@ void *UbsShmCallback(void* args) // a slow daemon cannot stall the timer thread for the whole list. SHM shm; { - LOCK_GUARD(shm_list->shm_lock); + BAIDU_SCOPED_LOCK(shm_list->shm_lock); if (shm_list->head == nullptr) { return nullptr; } @@ -431,7 +431,7 @@ void *UbsShmCallback(void* args) } if (shm.addr == nullptr) { LOG(ERROR) << "Ubs input shm param is invalid, addr is NULL."; - LOCK_GUARD(shm_list->shm_lock); + BAIDU_SCOPED_LOCK(shm_list->shm_lock); DeleteShmToList(shm_list); return nullptr; } @@ -444,19 +444,16 @@ void *UbsShmCallback(void* args) LOG(ERROR) << "Ubs unmap shm=" << shm.name << " length=" << shm.len << " failed, ret=" << ret; return nullptr; // node stays at head, retried } - LOG(INFO) << "Ubs unmap shm=" << shm.name << " length=" << shm.len << " success."; { - LOCK_GUARD(shm_list->shm_lock); + BAIDU_SCOPED_LOCK(shm_list->shm_lock); DeleteShmToList(shm_list); } ret = ubsmem_shmem_deallocate(shm.name); if (ret != UBSM_OK) { LOG(ERROR) << "Ubs delete shm=" << shm.name << " failed, ret=" << ret; - return nullptr; } - LOG(INFO) << "Ubs free local shm=" << shm.name << " length=" << shm.len << " success."; return nullptr; } @@ -465,7 +462,7 @@ RETURN_CODE UbsShmAddTimer(ShmList *shm_list) const uint64_t timer_interval_us = (uint64_t)FLAGS_ub_flying_io_timeout_s * SEC_TO_USEC; RETURN_CODE rc = UbrTimerStart(&g_shm_timer_id, 0, timer_interval_us, UbsShmCallback, (void*)shm_list); - if (UNLIKELY(rc != UBRING_OK)) { + if (BAIDU_UNLIKELY(rc != UBRING_OK)) { LOG(ERROR) << "Start shm timer failed."; return UBRING_ERR; } @@ -521,11 +518,11 @@ RETURN_CODE DestroyShmTimer(ShmList *shm_list) RETURN_CODE IsExistInShmList(ShmList *shm_list, const SHM *shm) { - if (UNLIKELY(shm_list == nullptr || shm == nullptr)) { + if (BAIDU_UNLIKELY(shm_list == nullptr || shm == nullptr)) { LOG(ERROR) << "Shm list or shm is null."; return UBRING_ERR; } - LOCK_GUARD(shm_list->shm_lock); + BAIDU_SCOPED_LOCK(shm_list->shm_lock); ShmListNode *cur_node = shm_list->head; while (cur_node != nullptr) { @@ -556,7 +553,7 @@ RETURN_CODE AddShmToList(ShmList *shm_list, SHM *shm) } memcpy(&new_shm_node->shm, shm, sizeof(SHM)); - LOCK_GUARD(shm_list->shm_lock); + BAIDU_SCOPED_LOCK(shm_list->shm_lock); new_shm_node->next = nullptr; new_shm_node->prev = shm_list->tail; if (shm_list->tail) { diff --git a/src/brpc/ubshm/timer/timer_mgr.cpp b/src/brpc/ubshm/timer/timer_mgr.cpp index d4d4e6d5de..912e331c1e 100644 --- a/src/brpc/ubshm/timer/timer_mgr.cpp +++ b/src/brpc/ubshm/timer/timer_mgr.cpp @@ -15,10 +15,10 @@ // specific language governing permissions and limitations // under the License. -#include #include #include "bthread/bthread.h" // bthread_usleep #include "bthread/unstable.h" // bthread_timer_add/del +#include "butil/atomicops.h" #include "butil/time.h" #include "brpc/ubshm/timer/timer_mgr.h" @@ -46,18 +46,18 @@ enum UbrTimerState { // All atomics are seq_cst so no interleaving can release a ref twice or // free the task while a callback or the starter still touches it. struct UbrTimerTask { - UbrTimerId* slot; - std::atomic id; + butil::atomic* slot; + butil::atomic id; void* (*cb)(void*); void* arg; UbrTimerBackoffFn backoff; uint64_t interval_us; // timer thread only bool periodic; - std::atomic state; // kStarting/kScheduled/kDead - std::atomic stopped; - std::atomic ref; - std::atomic join_pending; // a DelAndWait is waiting - std::atomic done; // refs hit zero, joiner frees + butil::atomic state; // kStarting/kScheduled/kDead + butil::atomic stopped; + butil::atomic ref; + butil::atomic join_pending; // a DelAndWait is waiting + butil::atomic done; // refs hit zero, joiner frees }; namespace { @@ -117,9 +117,7 @@ void UbrTimerOnFire(void* p) { // RMW and seq_cst does not order the store-buffer case -- ownership of // the slot is the single arbiter. UbrTimerId expected = task; - const bool owned = - __atomic_compare_exchange_n(task->slot, &expected, (UbrTimerId) nullptr, - false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); + const bool owned = task->slot->compare_exchange_strong(expected, nullptr); if (owned) { task->cb(task->arg); } @@ -129,20 +127,20 @@ void UbrTimerOnFire(void* p) { } } -UbrTimerTask* TakeOutTask(UbrTimerId* slot) { - return __atomic_exchange_n(slot, (UbrTimerId) nullptr, __ATOMIC_SEQ_CST); +UbrTimerTask* TakeOutTask(butil::atomic* slot) { + return slot->exchange(nullptr); } -RETURN_CODE TimerStartInternal(UbrTimerId* slot, uint64_t delay_us, +RETURN_CODE TimerStartInternal(butil::atomic* slot, uint64_t delay_us, uint64_t interval_us, void* (*cb)(void*), void* arg, UbrTimerBackoffFn backoff) { - if (UNLIKELY(slot == nullptr || cb == nullptr)) { + if (BAIDU_UNLIKELY(slot == nullptr || cb == nullptr)) { LOG(ERROR) << "Ubr timer start invalid argument, slot=" << slot; return UBRING_ERR; } UbrTimerTask* task = new (std::nothrow) UbrTimerTask(); - if (UNLIKELY(task == nullptr)) { + if (BAIDU_UNLIKELY(task == nullptr)) { LOG(ERROR) << "Fail to malloc ubring timer task."; return UBRING_ERR; } @@ -162,24 +160,20 @@ RETURN_CODE TimerStartInternal(UbrTimerId* slot, uint64_t delay_us, // Publish the real task before scheduling so a delete or a DelAndWait // racing the start always has an object to act on or wait for. UbrTimerId expected = nullptr; - if (!__atomic_compare_exchange_n(slot, &expected, (UbrTimerId) task, false, - __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) { + if (!slot->compare_exchange_strong(expected, task)) { LOG(ERROR) << "Ubr timer start refused, slot already occupied"; delete task; // never published return UBRING_ERR; } bthread_timer_t id = 0; - if (UNLIKELY(bthread_timer_add( + if (BAIDU_UNLIKELY(bthread_timer_add( &id, butil::microseconds_from_now((int64_t)delay_us), UbrTimerOnFire, task) != 0)) { LOG(ERROR) << "Fail to add ubring timer"; task->state.store(kDead); // wake DelAndWait waiters expected = task; - const bool owned = - __atomic_compare_exchange_n(slot, &expected, (UbrTimerId) nullptr, - false, __ATOMIC_SEQ_CST, - __ATOMIC_SEQ_CST); + const bool owned = slot->compare_exchange_strong(expected, nullptr); ReleaseRef(task); // schedule, never ran if (owned) { ReleaseRef(task); // owner @@ -201,13 +195,13 @@ RETURN_CODE TimerStartInternal(UbrTimerId* slot, uint64_t delay_us, } // namespace -RETURN_CODE UbrTimerStart(UbrTimerId* slot, uint64_t delay_us, +RETURN_CODE UbrTimerStart(butil::atomic* slot, uint64_t delay_us, uint64_t interval_us, void* (*cb)(void*), void* arg, UbrTimerBackoffFn backoff) { return TimerStartInternal(slot, delay_us, interval_us, cb, arg, backoff); } -int UbrTimerDel(UbrTimerId* slot) { +int UbrTimerDel(butil::atomic* slot) { if (slot == nullptr) { return 1; } @@ -242,7 +236,7 @@ int UbrTimerDel(UbrTimerId* slot) { // callback may still complete. } -void UbrTimerDelAndWait(UbrTimerId* slot) { +void UbrTimerDelAndWait(butil::atomic* slot) { if (slot == nullptr) { return; } diff --git a/src/brpc/ubshm/timer/timer_mgr.h b/src/brpc/ubshm/timer/timer_mgr.h index f4d244eceb..24ef2e0b15 100644 --- a/src/brpc/ubshm/timer/timer_mgr.h +++ b/src/brpc/ubshm/timer/timer_mgr.h @@ -39,8 +39,10 @@ typedef uint64_t (*UbrTimerBackoffFn)(void* arg, uint64_t cur_interval_us); // re-arm itself after every run until deleted. One-shot timers release // their handle slot before running the callback, so the callback may free // the object that stores the slot; the task object itself is released -// automatically. -RETURN_CODE UbrTimerStart(UbrTimerId* slot, uint64_t delay_us, +// automatically. `slot' must point to a real butil::atomic +// object, so start and delete can race on one RMW without reinterpreting +// plain storage as an atomic. +RETURN_CODE UbrTimerStart(butil::atomic* slot, uint64_t delay_us, uint64_t interval_us, void* (*cb)(void*), void* arg, UbrTimerBackoffFn backoff = nullptr); @@ -59,7 +61,7 @@ RETURN_CODE UbrTimerStart(UbrTimerId* slot, uint64_t delay_us, // Returns 1 when this caller did not acquire timer ownership. The callback, // another deleter, or the scheduling path is responsible for settling the // timer resources, so this caller must not reclaim them. -int UbrTimerDel(UbrTimerId* slot); +int UbrTimerDel(butil::atomic* slot); // Delete and wait until a possibly running callback finished, so the // caller can free resources reachable from `arg'. Never call this on the @@ -68,7 +70,7 @@ int UbrTimerDel(UbrTimerId* slot); // 1) and cannot be waited for through the slot. The wait polls with // bthread_usleep, which degrades to ::usleep on plain pthread callers // (e.g. process-exit paths). -void UbrTimerDelAndWait(UbrTimerId* slot); +void UbrTimerDelAndWait(butil::atomic* slot); } // namespace ubring } // namespace brpc diff --git a/src/brpc/ubshm/ub_endpoint.cpp b/src/brpc/ubshm/ub_endpoint.cpp index 089ccef5bd..51003d56a9 100644 --- a/src/brpc/ubshm/ub_endpoint.cpp +++ b/src/brpc/ubshm/ub_endpoint.cpp @@ -533,7 +533,7 @@ void* UBShmEndpoint::ProcessHandshakeAtServer(void* arg) { } int result = snprintf(local_trx_shm.name, SHM_MAX_NAME_BUFF_LEN, "%s_%s", client_name, SERVER_SHM_NAME_SUFFIX); - if (UNLIKELY(result < 0)) { + if (BAIDU_UNLIKELY(result < 0)) { LOG(WARNING) << "Copy client shared memory name failed, ret=" << result; ub_transport->_ub_state = UBShmTransport::UB_OFF; } @@ -641,14 +641,14 @@ ssize_t UBShmEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { ssize_t nw = 0; errno = 0; nw = _ub_ring->UbrTrxWritev(vec, nvec); - if (UNLIKELY(nw == -1)) { + if (BAIDU_UNLIKELY(nw == -1)) { if (errno == EMSGSIZE) { LOG(ERROR) << "Non-blocking send msg failed, message is larger than ubring capacity."; } else { LOG(ERROR) << "Non-blocking send msg in failed, connection has been closed."; errno = EPIPE; } - } else if (UNLIKELY(nw == UBRING_RETRY)) { + } else if (BAIDU_UNLIKELY(nw == UBRING_RETRY)) { errno = EAGAIN; nw = -1; } @@ -833,7 +833,7 @@ int UBShmEndpoint::PollingModeInitialize(bthread_tag_t tag, } struct FnArgs { Poller* poller; - std::atomic* running; + butil::atomic* running; }; auto fn = [](void* p) -> void* { std::unique_ptr args(static_cast(p)); diff --git a/src/brpc/ubshm/ub_endpoint.h b/src/brpc/ubshm/ub_endpoint.h index a29a0927f7..c93fa330de 100644 --- a/src/brpc/ubshm/ub_endpoint.h +++ b/src/brpc/ubshm/ub_endpoint.h @@ -234,7 +234,7 @@ friend class Socket; struct BAIDU_CACHELINE_ALIGNMENT PollerGroup { PollerGroup() : pollers(FLAGS_ub_poller_num), running(false) {} std::vector pollers; - std::atomic running; + butil::atomic running; }; static std::vector _poller_groups; diff --git a/src/brpc/ubshm/ub_ring.cpp b/src/brpc/ubshm/ub_ring.cpp index 25b6f2749b..3ba1af8aa8 100644 --- a/src/brpc/ubshm/ub_ring.cpp +++ b/src/brpc/ubshm/ub_ring.cpp @@ -56,12 +56,12 @@ UBRing::~UBRing() RETURN_CODE UBRing::UbrTrxMapShm(SHM *local_shm, SHM *remote_shm) { RETURN_CODE rc = UbrTrxMapLocalShm(local_shm); - if (UNLIKELY(rc != UBRING_OK)) { + if (BAIDU_UNLIKELY(rc != UBRING_OK)) { LOG(ERROR) << "Trx map local shared memory failed."; return rc; } rc = UbrTrxMapRemoteShm(remote_shm); - if (UNLIKELY(rc != UBRING_OK)) { + if (BAIDU_UNLIKELY(rc != UBRING_OK)) { LOG(ERROR) << "Trx map remote shared memory failed."; return rc; } @@ -69,10 +69,10 @@ RETURN_CODE UBRing::UbrTrxMapShm(SHM *local_shm, SHM *remote_shm) } static void UbrDoAsynClearWork(UbrTrx *trx, uint64_t expect_ubr_id) { - if (UNLIKELY(UBRing::UbrTrxFreeShm(trx) != UBRING_OK)) { + if (BAIDU_UNLIKELY(UBRing::UbrTrxFreeShm(trx) != UBRING_OK)) { LOG(ERROR) << "Trx close, wait for local shm " << trx->local_shm.name << " free fail."; } - if (UNLIKELY(UBRingManager::ReleaseUbrTrxFromMgr(trx, expect_ubr_id) != UBRING_OK)) { + if (BAIDU_UNLIKELY(UBRingManager::ReleaseUbrTrxFromMgr(trx, expect_ubr_id) != UBRING_OK)) { LOG(ERROR) << "Trx close, release shm " << trx->local_shm.name << " trx failed."; } } @@ -88,7 +88,7 @@ static void UbrDoPassiveClearWork(UbrTrx *trx, uint64_t expect_ubr_id) { LOG(ERROR) << "Trx passive clear, delete local shm " << trx->local_shm.name << " failed. ret=" << rc; } - if (UNLIKELY(UBRingManager::ReleaseUbrTrxFromMgr(trx, expect_ubr_id) != UBRING_OK)) { + if (BAIDU_UNLIKELY(UBRingManager::ReleaseUbrTrxFromMgr(trx, expect_ubr_id) != UBRING_OK)) { LOG(ERROR) << "Trx passive clear, release shm " << trx->local_shm.name << " trx failed."; } } @@ -99,14 +99,14 @@ static void UbrDoPassiveClearWork(UbrTrx *trx, uint64_t expect_ubr_id) { // body, used directly when the timer cannot be started. static RETURN_CODE UbrScheduleClearTimer(UbrTrx *trx, void* (*cb)(void*), void (*work)(UbrTrx*, uint64_t)) { - if (UNLIKELY(trx == nullptr || trx->local_shm.addr == nullptr)) { + if (BAIDU_UNLIKELY(trx == nullptr || trx->local_shm.addr == nullptr)) { return UBRING_OK; // released trx, stale event } - if (__atomic_load_n(&trx->cleanup_ctl, __ATOMIC_SEQ_CST) != nullptr) { + if (trx->cleanup_ctl.load() != nullptr) { return UBRING_OK; // cleanup already scheduled } auto* ctl = new (std::nothrow) UbrCleanupCtl(); - if (UNLIKELY(ctl == nullptr)) { + if (BAIDU_UNLIKELY(ctl == nullptr)) { LOG(ERROR) << "Fail to malloc ubr cleanup ctl."; return UBRING_ERR; } @@ -119,8 +119,7 @@ static RETURN_CODE UbrScheduleClearTimer(UbrTrx *trx, void* (*cb)(void*), // TryPublishUnitCleanupCtl UbrCleanupCtl* expected = nullptr; - if (!__atomic_compare_exchange_n(&trx->cleanup_ctl, &expected, ctl, false, - __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) { + if (!trx->cleanup_ctl.compare_exchange_strong(expected, ctl)) { delete ctl; // another schedule won return UBRING_OK; } @@ -130,16 +129,14 @@ static RETURN_CODE UbrScheduleClearTimer(UbrTrx *trx, void* (*cb)(void*), // anchor: force close or the new occupant owns it now. Nothing // was armed yet -- just undo the trx-side publication. expected = ctl; - __atomic_compare_exchange_n(&trx->cleanup_ctl, &expected, - (UbrCleanupCtl*) nullptr, false, - __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); + trx->cleanup_ctl.compare_exchange_strong(expected, nullptr); ctl->ReleaseRef(); // timer/callback reference, never armed ctl->ReleaseRef(); // starter reference return UBRING_OK; } RETURN_CODE rc = UbrTimerStart(&ctl->timer, (uint64_t)FLAGS_ub_flying_io_timeout_s * SEC_TO_USEC, 0, cb, ctl); - if (UNLIKELY(rc != UBRING_OK)) { + if (BAIDU_UNLIKELY(rc != UBRING_OK)) { // The timer was never scheduled: this path owns the manager, // timer/callback and starter references. Roll the schedule back // and run the cleanup inline so the trx does not end up with @@ -156,24 +153,20 @@ static RETURN_CODE UbrScheduleClearTimer(UbrTrx *trx, void* (*cb)(void*), // close that this cleanup is owned (RUNNING/DONE), otherwise it // would fall into its no-ctl branch and clean the trx again. UbrCleanupCtl* published = ctl; - __atomic_compare_exchange_n(&trx->cleanup_ctl, &published, - (UbrCleanupCtl*) nullptr, false, - __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); + trx->cleanup_ctl.compare_exchange_strong(published, nullptr); UBRingManager::DetachUnitCleanupCtl(trx->trx_mgr_index, ctl); } ctl->ReleaseRef(); // timer/callback reference ctl->ReleaseRef(); // starter reference return UBRING_ERR; } - if (UNLIKELY(ATOMIC_LOAD(trx->ubr_id) != ctl->ubr_id)) { + if (BAIDU_UNLIKELY(ATOMIC_LOAD(trx->ubr_id) != ctl->ubr_id)) { // Published onto a slot that was released and reused meanwhile. if (UbrTimerDel(&ctl->timer) == 0) { ctl->ReleaseRef(); // timer/callback reference } UbrCleanupCtl* published = ctl; - if (!__atomic_compare_exchange_n(&trx->cleanup_ctl, &published, - (UbrCleanupCtl*) nullptr, false, - __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) { + if (!trx->cleanup_ctl.compare_exchange_strong(published, nullptr)) { published = nullptr; } UBRingManager::DetachUnitCleanupCtl(trx->trx_mgr_index, ctl); @@ -186,7 +179,7 @@ static RETURN_CODE UbrScheduleClearTimer(UbrTrx *trx, void* (*cb)(void*), RETURN_CODE UBRing::UbrTrxClose() { RETURN_CODE close_check_rc = UbrTrxCloseCheck(_trx); - if (UNLIKELY(close_check_rc != UBRING_OK)) { + if (BAIDU_UNLIKELY(close_check_rc != UBRING_OK)) { if (close_check_rc == UBRING_REENTRY) { LOG(INFO) << "Trx close skipped, already closing, local name=" << _trx->local_shm.name; return UBRING_OK; @@ -242,10 +235,10 @@ RETURN_CODE UBRing::UbrTrxClose() { if (_trx->ubr_tx.remote_rx_event_q.addr != nullptr) { ((UbrEventQMsg *)_trx->ubr_tx.remote_rx_event_q.addr)->flag = UBR_STATE_CLOSED; } - if (UNLIKELY(UbrTrxFreeShm(_trx) != UBRING_OK)) { + if (BAIDU_UNLIKELY(UbrTrxFreeShm(_trx) != UBRING_OK)) { LOG(WARNING) << "Force close, local shm " << _trx->local_shm.name << " free failed."; } - if (UNLIKELY(UBRingManager::ReleaseUbrTrxFromMgr(_trx, expect_ubr_id) != UBRING_OK)) { + if (BAIDU_UNLIKELY(UBRingManager::ReleaseUbrTrxFromMgr(_trx, expect_ubr_id) != UBRING_OK)) { LOG(WARNING) << "Force close, release trx " << _trx->local_shm.name << " failed."; } if (ctl != nullptr) { @@ -261,7 +254,7 @@ RETURN_CODE UBRing::UbrTrxClose() { } _trx->ubr_rx.trx_state = UBR_STATE_CLOSED; RETURN_CODE rc; - if (UNLIKELY((rc = ClearTrxResource(_trx)) != UBRING_OK)) { + if (BAIDU_UNLIKELY((rc = ClearTrxResource(_trx)) != UBRING_OK)) { if (rc == UBRING_REENTRY) { LOG(INFO) << "Trx close, peer is closing, trx local name=" << _trx->local_shm.name; return UBRING_OK; @@ -286,7 +279,7 @@ static uint64_t UbrCloseTimerBackoff(void* arg, uint64_t cur_interval_us) { auto* local_tx_event_q = (UbrEventQMsg *)trx->ubr_tx.local_tx_event_q.addr; const uint64_t in_io_id = ATOMIC_LOAD(trx->ubr_rx.in_io_id); const uint64_t out_io_id = ATOMIC_LOAD(trx->ubr_tx.out_io_id); - if (UNLIKELY(local_rx_event_q == nullptr)) { + if (BAIDU_UNLIKELY(local_rx_event_q == nullptr)) { return (uint64_t)FLAGS_ub_event_queue_timer_interval_us; } const bool has_traffic = (in_io_id != trx->close_chk_in_io_id) || @@ -308,7 +301,7 @@ static uint64_t UbrCloseTimerBackoff(void* arg, uint64_t cur_interval_us) { } RETURN_CODE UBRing::UbrAddCloseTimer() { - if (UNLIKELY(_trx == nullptr)) { + if (BAIDU_UNLIKELY(_trx == nullptr)) { LOG(ERROR) << "Trx add close timer failed, trx is null."; return UBRING_ERR; } @@ -319,7 +312,7 @@ RETURN_CODE UBRing::UbrAddCloseTimer() { RETURN_CODE rc = UbrTimerStart(&_trx->close_timer, 0, interval_us, UbrTrxCloseCallback, (void*)_trx, UbrCloseTimerBackoff); - if (UNLIKELY(rc != UBRING_OK)) { + if (BAIDU_UNLIKELY(rc != UBRING_OK)) { LOG(ERROR) << "Start ubr close timer failed, trx local name=" << _trx->local_shm.name; return UBRING_ERR; } @@ -327,12 +320,12 @@ RETURN_CODE UBRing::UbrAddCloseTimer() { } RETURN_CODE UBRing::UbrAddTimer() { - if (UNLIKELY(UbrAddCloseTimer() != UBRING_OK)) { + if (BAIDU_UNLIKELY(UbrAddCloseTimer() != UBRING_OK)) { LOG(ERROR) << "Ubr " << _trx->local_shm.name << " add closed timer failed."; return UBRING_ERR; } - if (UNLIKELY(UbrAddHBTimer() != UBRING_OK)) { + if (BAIDU_UNLIKELY(UbrAddHBTimer() != UBRING_OK)) { UbrTimerDelAndWait(&_trx->close_timer); LOG(ERROR) << "Ubr " << _trx->local_shm.name << " add heartbeat timer failed."; return UBRING_ERR; @@ -342,7 +335,7 @@ RETURN_CODE UBRing::UbrAddTimer() { void* UBRing::UbrTrxCloseCallback(void* args) { auto* trx = (UbrTrx*) args; - if (UNLIKELY(UBRing::UbrTrxCallbackCheck(trx) != UBRING_OK)) { + if (BAIDU_UNLIKELY(UBRing::UbrTrxCallbackCheck(trx) != UBRING_OK)) { return nullptr; } @@ -369,7 +362,7 @@ void* UBRing::UbrTrxCloseCallback(void* args) { } remote_rx_event_q->flag = UBR_STATE_CLOSED; RETURN_CODE clear_rc = ClearTrxResource(trx); - if (UNLIKELY(clear_rc != UBRING_OK && clear_rc != UBRING_REENTRY)) { + if (BAIDU_UNLIKELY(clear_rc != UBRING_OK && clear_rc != UBRING_REENTRY)) { LOG(ERROR) << "Trx close callback failed, " << trx->local_shm.name << " clear trx resource failed."; break; } @@ -378,7 +371,7 @@ void* UBRing::UbrTrxCloseCallback(void* args) { } RETURN_CODE UBRing::UbrAddHBTimer() { - if (UNLIKELY(_trx == nullptr)) { + if (BAIDU_UNLIKELY(_trx == nullptr)) { LOG(ERROR) << "Trx add heartbeat timer failed, trx is null."; return UBRING_ERR; } @@ -386,7 +379,7 @@ RETURN_CODE UBRing::UbrAddHBTimer() { const uint64_t interval_us = (uint64_t)FLAGS_ub_hb_timer_interval_s * SEC_TO_USEC; RETURN_CODE rc = UbrTimerStart(&_trx->hb_timer, 0, interval_us, UbrTrxHBCallback, (void*)_trx); - if (UNLIKELY(rc != UBRING_OK)) { + if (BAIDU_UNLIKELY(rc != UBRING_OK)) { LOG(ERROR) << "Start ubr heartbeat timer failed."; return UBRING_ERR; } @@ -395,7 +388,7 @@ RETURN_CODE UBRing::UbrAddHBTimer() { RETURN_CODE UBRing::UbrPassiveClearTrx(UbrTrx *trx) { RETURN_CODE passive_close_check_rc = UbrTrxCloseCheck(trx); - if (UNLIKELY(passive_close_check_rc != UBRING_OK)) { + if (BAIDU_UNLIKELY(passive_close_check_rc != UBRING_OK)) { if (passive_close_check_rc == UBRING_REENTRY) { LOG(INFO) << "Passive close skipped, active close in progress, name=" << trx->local_shm.name; return ClearTrxResource(trx); @@ -414,7 +407,7 @@ RETURN_CODE UBRing::UbrPassiveClearTrx(UbrTrx *trx) { void* UBRing::UbrPassiveClearCallback(void* args) { auto* ctl = (UbrCleanupCtl*)args; - if (UNLIKELY(ctl == nullptr)) { + if (BAIDU_UNLIKELY(ctl == nullptr)) { LOG(ERROR) << "Trx passive clear callback failed, ctl is null."; return nullptr; } @@ -427,7 +420,7 @@ void* UBRing::UbrPassiveClearCallback(void* args) { return nullptr; } UbrTrx* trx = ctl->trx; - if (UNLIKELY(UBRingManager::IsUbrTrxSlotUsed(trx->trx_mgr_index, ctl->ubr_id))) { + if (BAIDU_UNLIKELY(UBRingManager::IsUbrTrxSlotUsed(trx->trx_mgr_index, ctl->ubr_id))) { UbrDoPassiveClearWork(trx, ctl->ubr_id); } ATOMIC_STORE(ctl->state, UBR_CLEANUP_DONE); @@ -437,13 +430,13 @@ void* UBRing::UbrPassiveClearCallback(void* args) { void* UBRing::UbrTrxHBCallback(void* args) { auto* trx = (UbrTrx*) args; - if (UNLIKELY(UbrTrxCallbackCheck(trx) != UBRING_OK)) { + if (BAIDU_UNLIKELY(UbrTrxCallbackCheck(trx) != UBRING_OK)) { return nullptr; } auto* local_data_status = (UbrDataStatusQMsg *)trx->ubr_tx.local_data_status_q.addr; auto* remote_data_status = (UbrDataStatusQMsg *)trx->ubr_rx.remote_data_status_q.addr; - if (UNLIKELY(local_data_status == nullptr || remote_data_status == nullptr)) { + if (BAIDU_UNLIKELY(local_data_status == nullptr || remote_data_status == nullptr)) { LOG(ERROR) << "Heartbeat error, datastatus is NULL."; return nullptr; } @@ -473,7 +466,7 @@ void* UBRing::UbrTrxHBCallback(void* args) { } RETURN_CODE UBRing::UbrAddAsynClearTimer(UbrTrx *trx) { - if (UNLIKELY(trx == nullptr)) { + if (BAIDU_UNLIKELY(trx == nullptr)) { LOG(ERROR) << "Trx add close timer failed, trx is null."; return UBRING_ERR; } @@ -483,7 +476,7 @@ RETURN_CODE UBRing::UbrAddAsynClearTimer(UbrTrx *trx) { void *UBRing::UbrAsynClearCallback(void *args) { auto* ctl = (UbrCleanupCtl*) args; - if (UNLIKELY(ctl == nullptr)) { + if (BAIDU_UNLIKELY(ctl == nullptr)) { LOG(ERROR) << "Trx close, ctl is null."; return nullptr; } @@ -496,7 +489,7 @@ void *UBRing::UbrAsynClearCallback(void *args) return nullptr; } UbrTrx* trx = ctl->trx; - if (UNLIKELY(UBRingManager::IsUbrTrxSlotUsed(trx->trx_mgr_index, ctl->ubr_id))) { + if (BAIDU_UNLIKELY(UBRingManager::IsUbrTrxSlotUsed(trx->trx_mgr_index, ctl->ubr_id))) { UbrDoAsynClearWork(trx, ctl->ubr_id); } ATOMIC_STORE(ctl->state, UBR_CLEANUP_DONE); @@ -506,7 +499,7 @@ void *UBRing::UbrAsynClearCallback(void *args) int UBRing::UbrTrxSend(const void *buf, uint32_t buf_len) { - if (UNLIKELY(CheckTrxSendPreCheck(_trx) != UBRING_OK)) { + if (BAIDU_UNLIKELY(CheckTrxSendPreCheck(_trx) != UBRING_OK)) { return UBRING_ERR; } // 1.2 Calculate space @@ -549,7 +542,7 @@ int UBRing::UbrTrxSend(const void *buf, uint32_t buf_len) int UBRing::UbrTrxRecv(void *buf, uint32_t buf_len) { RETURN_CODE rc = UBRING_OK; - if (UNLIKELY((rc = CheckTrxRecvParam(_trx, buf, buf_len)) != UBRING_OK)) { + if (BAIDU_UNLIKELY((rc = CheckTrxRecvParam(_trx, buf, buf_len)) != UBRING_OK)) { return (rc == UBR_NOT_CONNECTED) ? 0 : rc; } UbrMsgFormat *data_msg = (UbrMsgFormat *)_trx->ubr_rx.local_data_q.addr; @@ -564,7 +557,7 @@ int UBRing::UbrTrxRecv(void *buf, uint32_t buf_len) int UBRing::UbrTrxRecvBlockMode(uint8_t *dest, uint32_t buf_len) { RETURN_CODE rc = UBRING_OK; - if (UNLIKELY((rc = CheckTrxRecvParam(_trx, dest, buf_len)) != UBRING_OK)) { + if (BAIDU_UNLIKELY((rc = CheckTrxRecvParam(_trx, dest, buf_len)) != UBRING_OK)) { return (rc == UBR_NOT_CONNECTED) ? 0 : rc; } @@ -577,7 +570,7 @@ int UBRing::UbrTrxRecvBlockMode(uint8_t *dest, uint32_t buf_len) bool need_update_epoll_eof_pos = ubr_rx->read_pos == ubr_rx->ep_eof_pos; while (not_eof_encountered && remaining_len > 0) { - if (UNLIKELY(CheckTrxRecvPreCheck(_trx) != UBRING_OK)) { + if (BAIDU_UNLIKELY(CheckTrxRecvPreCheck(_trx) != UBRING_OK)) { return UBRING_ERR; } UbrMsgFormat *current_chunk = &data_msg[ubr_rx->read_pos]; @@ -594,7 +587,7 @@ int UBRing::UbrTrxRecvBlockMode(uint8_t *dest, uint32_t buf_len) } uint8_t chunk_msg_len = current_chunk->header[UBR_MSG_LEN_INDEX]; uint8_t cur_index = current_chunk->header[UBR_MSG_CUR_INDEX]; - if (UNLIKELY(!IsRecvChunkHeaderValid(chunk_msg_len, cur_index))) { + if (BAIDU_UNLIKELY(!IsRecvChunkHeaderValid(chunk_msg_len, cur_index))) { LOG(ERROR) << "Trx recv failed, invalid chunk header msg_len=" << (uint32_t)chunk_msg_len << " cur_index=" << (uint32_t)cur_index; errno = EBADMSG; @@ -607,7 +600,7 @@ int UBRing::UbrTrxRecvBlockMode(uint8_t *dest, uint32_t buf_len) total_copied += copy_len; remaining_len -= copy_len; current_chunk->header[UBR_MSG_CUR_INDEX] += (uint8_t)copy_len; - if (LIKELY(current_chunk->header[UBR_MSG_CUR_INDEX] == chunk_msg_len)) { + if (BAIDU_LIKELY(current_chunk->header[UBR_MSG_CUR_INDEX] == chunk_msg_len)) { current_chunk->header[UBR_MSG_FLAG_INDEX] = UBR_MSG_CHUNK_NONE; UpdateDataQTail(_trx); ubr_rx->read_pos = (ubr_rx->read_pos + 1) % ubr_rx->capacity; @@ -621,7 +614,7 @@ int UBRing::UbrTrxRecvBlockMode(uint8_t *dest, uint32_t buf_len) ssize_t UBRing::UbrTrxWritev(const struct iovec *iov, int iovcnt) { - if (UNLIKELY(CheckTrxSendPreCheck(_trx) != UBRING_OK)) { + if (BAIDU_UNLIKELY(CheckTrxSendPreCheck(_trx) != UBRING_OK)) { return UBRING_ERR; } @@ -678,7 +671,7 @@ ssize_t UBRing::UbrTrxWritev(const struct iovec *iov, int iovcnt) ssize_t UBRing::UbrTrxReadv(const struct iovec *iov, int iovcnt) { RETURN_CODE rc = UBRING_OK; - if (UNLIKELY((rc = CheckTrxRecvParam(_trx, iov, (uint32_t)iovcnt)) != UBRING_OK)) { + if (BAIDU_UNLIKELY((rc = CheckTrxRecvParam(_trx, iov, (uint32_t)iovcnt)) != UBRING_OK)) { return (rc == UBR_NOT_CONNECTED) ? 0 : rc; } UbrMsgFormat *data_msg = (UbrMsgFormat *)_trx->ubr_rx.local_data_q.addr; @@ -689,7 +682,7 @@ ssize_t UBRing::UbrTrxReadv(const struct iovec *iov, int iovcnt) return -1; } ssize_t nr = UbrTrxReadvBlockMode(iov, iovcnt); - if (UNLIKELY(nr == -1)) { + if (BAIDU_UNLIKELY(nr == -1)) { LOG(ERROR) << "Non-blocking readv msg in failed, connection has been closed."; errno = EPIPE; return -1; @@ -700,7 +693,7 @@ ssize_t UBRing::UbrTrxReadv(const struct iovec *iov, int iovcnt) ssize_t UBRing::UbrTrxReadvBlockMode(const struct iovec *iov, int iovcnt) { RETURN_CODE rc = UBRING_OK; - if (UNLIKELY((rc = CheckTrxRecvParam(_trx, iov, (uint32_t)iovcnt)) != UBRING_OK)) { + if (BAIDU_UNLIKELY((rc = CheckTrxRecvParam(_trx, iov, (uint32_t)iovcnt)) != UBRING_OK)) { return (rc == UBR_NOT_CONNECTED) ? 0 : rc; } @@ -720,15 +713,15 @@ ssize_t UBRing::UbrTrxReadvBlockMode(const struct iovec *iov, int iovcnt) RETURN_CODE UBRing::IsUbrTrxReadable(uint32_t ep_event) { - if (UNLIKELY(_trx == nullptr)) { + if (BAIDU_UNLIKELY(_trx == nullptr)) { LOG(ERROR) << "The trx to be checked is NULL."; return UBRING_ERR; } - if (UNLIKELY(_trx->local_shm.addr == nullptr)) { + if (BAIDU_UNLIKELY(_trx->local_shm.addr == nullptr)) { LOG(ERROR) << "The trx local_shm to be checked is NULL."; return UBRING_ERR; } - if (UNLIKELY(_trx->ubr_tx.trx_state != UBR_STATE_CONNECTED)) { + if (BAIDU_UNLIKELY(_trx->ubr_tx.trx_state != UBR_STATE_CONNECTED)) { return UBRING_ERR; } @@ -755,24 +748,24 @@ RETURN_CODE UBRing::IsUbrTrxReadable(uint32_t ep_event) RETURN_CODE UBRing::IsUbrTrxWriteable(uint32_t ep_event) { - if (UNLIKELY(_trx == nullptr)) { + if (BAIDU_UNLIKELY(_trx == nullptr)) { LOG(ERROR) << "The trx to be checked is NULL."; return UBRING_ERR; } - if (UNLIKELY(_trx->local_shm.addr == nullptr)) { + if (BAIDU_UNLIKELY(_trx->local_shm.addr == nullptr)) { LOG(ERROR) << "The trx local_shm to be checked is NULL."; return UBRING_ERR; } - if (UNLIKELY((UbrEventQMsg *)_trx->ubr_tx.local_tx_event_q.addr == nullptr)) { + if (BAIDU_UNLIKELY((UbrEventQMsg *)_trx->ubr_tx.local_tx_event_q.addr == nullptr)) { LOG(ERROR) << "The trx local_tx_event_q addr is NULL."; return UBRING_ERR; } - if (UNLIKELY((UbrEventQMsg *)_trx->ubr_tx.local_data_status_q.addr == nullptr)) { + if (BAIDU_UNLIKELY((UbrEventQMsg *)_trx->ubr_tx.local_data_status_q.addr == nullptr)) { LOG(ERROR) << "The trx local_data_status_q addr is NULL."; return UBRING_ERR; } - if (UNLIKELY(_trx->ubr_tx.trx_state != UBR_STATE_CONNECTED)) { + if (BAIDU_UNLIKELY(_trx->ubr_tx.trx_state != UBR_STATE_CONNECTED)) { LOG(ERROR) << "The trx is not connected state."; return UBRING_ERR; } @@ -816,13 +809,13 @@ RETURN_CODE UBRing::UbrTrxFreeShm(UbrTrx *trx) RETURN_CODE rc = UBRING_OK; rc = ShmMunmap(&trx->local_shm); - if (UNLIKELY(rc != UBRING_OK)) { + if (BAIDU_UNLIKELY(rc != UBRING_OK)) { LOG(ERROR) << "Trx close, local unmap " << trx->local_shm.name << " shm fail."; return UBRING_ERR; } rc = ShmFree(&trx->local_shm); - if (UNLIKELY(rc != UBRING_OK)) { + if (BAIDU_UNLIKELY(rc != UBRING_OK)) { if (rc != SHM_ERR_RESOURCE_ATTACHED && rc != SHM_ERR_NOT_FOUND) { LOG(ERROR) << "Wait for " << trx->local_shm.name << " local shm free fail."; return UBRING_ERR; @@ -843,7 +836,7 @@ RETURN_CODE UBRing::UbrTrxFreeShm(UbrTrx *trx) RETURN_CODE UBRing::UbrUnlinkLocalShm() { - if (UNLIKELY(_trx == nullptr)) { + if (BAIDU_UNLIKELY(_trx == nullptr)) { return UBRING_ERR; } RETURN_CODE rc = ShmFree(&_trx->local_shm); @@ -896,11 +889,11 @@ void UBRing::PrewriteUbrRx(UbrRx *rx) RETURN_CODE UBRing::UbrTrxMapLocalShm(SHM *local_shm) { - if (UNLIKELY(_trx == nullptr)) { + if (BAIDU_UNLIKELY(_trx == nullptr)) { LOG(ERROR) << "Trx map Shared memory failed, trx is null."; return UBRING_ERR; } - if (UNLIKELY(local_shm == nullptr || local_shm->addr == nullptr)) { + if (BAIDU_UNLIKELY(local_shm == nullptr || local_shm->addr == nullptr)) { LOG(ERROR) << "Trx map Shared memory failed, local_shm is null or addr is NULL."; return UBRING_ERR; } @@ -919,11 +912,11 @@ RETURN_CODE UBRing::UbrTrxMapLocalShm(SHM *local_shm) RETURN_CODE UBRing::UbrTrxMapRemoteShm(SHM *remote_shm) { - if (UNLIKELY(_trx == nullptr)) { + if (BAIDU_UNLIKELY(_trx == nullptr)) { LOG(ERROR) << "Trx map Shared memory failed, trx is null."; return UBRING_ERR; } - if (UNLIKELY(remote_shm == nullptr || remote_shm->addr == nullptr)) { + if (BAIDU_UNLIKELY(remote_shm == nullptr || remote_shm->addr == nullptr)) { LOG(ERROR) << "Trx map Shared memory failed, remote_shm is null or addr is NULL."; return UBRING_ERR; } @@ -943,7 +936,7 @@ RETURN_CODE UBRing::UbrTrxMapRemoteShm(SHM *remote_shm) RETURN_CODE UBRing::UbrServerTrxInit(SHM *local_shm, SHM *remote_shm) { RETURN_CODE rc = UbrTrxMapShm(local_shm, remote_shm); - if (UNLIKELY(rc != UBRING_OK)) { + if (BAIDU_UNLIKELY(rc != UBRING_OK)) { LOG(ERROR) <<"Trx map shared memory failed."; return rc; } @@ -953,7 +946,7 @@ RETURN_CODE UBRing::UbrServerTrxInit(SHM *local_shm, SHM *remote_shm) _trx->ubr_rx.capacity = local_data_msg_cap; _trx->ubr_tx.capacity = remote_data_msg_cap; rc = UBRingManager::GetUbrDealMsgMaxCnt(_trx->ubr_rx.capacity, &_trx->ubr_rx.deal_msg_max_cnt); - if (UNLIKELY(rc != UBRING_OK)) { + if (BAIDU_UNLIKELY(rc != UBRING_OK)) { LOG(ERROR) << "Get ubring deal msg max cnt."; return rc; } @@ -963,7 +956,7 @@ RETURN_CODE UBRing::UbrServerTrxInit(SHM *local_shm, SHM *remote_shm) ((UbrDataStatusQMsg *)(_trx->ubr_tx.local_data_status_q.addr))->tail = remote_data_msg_cap - 1; ((UbrDataStatusQMsg *)(_trx->ubr_rx.remote_data_status_q.addr))->tail = local_data_msg_cap - 1; - if (UNLIKELY(UbrAddTimer() != UBRING_OK)) { + if (BAIDU_UNLIKELY(UbrAddTimer() != UBRING_OK)) { LOG(ERROR) << "Ubr add timer failed, local_name=" << local_shm->name; return UBRING_ERR; } @@ -982,26 +975,26 @@ RETURN_CODE UBRing::UbrServerTrxInit(SHM *local_shm, SHM *remote_shm) int UBRing::UbrAllocateServerShm(SHM* remote_trx_shm, SHM* local_trx_shm) { UbrSetSleepTask(UBR_TASK_ACCEPT_MAP_FRONT); - if (UNLIKELY((ShmRemoteMalloc(remote_trx_shm)) != UBRING_OK)) { + if (BAIDU_UNLIKELY((ShmRemoteMalloc(remote_trx_shm)) != UBRING_OK)) { LOG(ERROR) << "Trx apply remote shared memory failed."; return -1; } - if (UNLIKELY((ShmLocalCalloc(local_trx_shm)) != UBRING_OK)) { + if (BAIDU_UNLIKELY((ShmLocalCalloc(local_trx_shm)) != UBRING_OK)) { LOG(ERROR) << "Trx apply local shared memory failed."; ShmRemoteFree(remote_trx_shm); return -1; } UbrTrx **ubr_trx_ptr = &_trx; - if (UNLIKELY((UBRingManager::AcquireUbrTrxFromMgr(ubr_trx_ptr)) != UBRING_OK)) { + if (BAIDU_UNLIKELY((UBRingManager::AcquireUbrTrxFromMgr(ubr_trx_ptr)) != UBRING_OK)) { LOG(ERROR) << "Acquire ubrtrx failed."; ShmRemoteFree(remote_trx_shm); ShmLocalFree(local_trx_shm); return -1; } _trx->type = TCP_TRX; - if (UNLIKELY((UbrServerTrxInit(local_trx_shm, remote_trx_shm)) != UBRING_OK)) { + if (BAIDU_UNLIKELY((UbrServerTrxInit(local_trx_shm, remote_trx_shm)) != UBRING_OK)) { LOG(ERROR) << "Server trx init failed."; UbrTrxFreeShm(_trx); UBRingManager::ReleaseUbrTrxFromMgr(_trx, ATOMIC_LOAD(_trx->ubr_id)); @@ -1013,13 +1006,13 @@ int UBRing::UbrAllocateServerShm(SHM* remote_trx_shm, SHM* local_trx_shm) { int UBRing::UbrAllocateLocalShm(SHM *local_trx_shm, const char *shm_name) { - if (UNLIKELY((UBRingManager::AcquireUbrTrxFromMgr(&(_trx))) != UBRING_OK)) { + if (BAIDU_UNLIKELY((UBRingManager::AcquireUbrTrxFromMgr(&(_trx))) != UBRING_OK)) { LOG(ERROR) << "Acquire ubrtrx failed, local_name=" << shm_name; return -1; } _trx->type = TCP_TRX; - if (UNLIKELY((ApplyAndMapLocalShm(local_trx_shm, shm_name)) != UBRING_OK)) { + if (BAIDU_UNLIKELY((ApplyAndMapLocalShm(local_trx_shm, shm_name)) != UBRING_OK)) { LOG(ERROR) << "Trx apply or map local shared memory failed, local_name=" << shm_name; _trx = nullptr; return -1; @@ -1030,7 +1023,7 @@ int UBRing::UbrAllocateLocalShm(SHM *local_trx_shm, const char *shm_name) int UBRing::UbrMapRemoteShm(SHM *local_trx_shm, const char *local_name) { RETURN_CODE rc = UbrMapRemoteShmAddTimer(local_trx_shm, local_name); - if (UNLIKELY(rc != UBRING_OK)) { + if (BAIDU_UNLIKELY(rc != UBRING_OK)) { LOG(ERROR) << "Connect Trx failed, local shm name=" << local_trx_shm->name; return -1; } @@ -1056,19 +1049,19 @@ RETURN_CODE UBRing::UbrMapRemoteShmAddTimer(SHM *local_trx_shm, const char *loca SHM_NAME_PREFIX, local_name, SERVER_SHM_NAME_SUFFIX); - if (UNLIKELY(result < 0)) { + if (BAIDU_UNLIKELY(result < 0)) { LOG(ERROR) << "Copy server shared memory name failed, local_name=" << local_name << ", ret=" << result; return UBRING_ERR; } UbrSetSleepTask(UBR_TASK_CONNECT_MAP_FRONT); RETURN_CODE rc = ApplyAndMapRemoteShm(&remote_trx_shm); - if (UNLIKELY(rc != UBRING_OK)) { + if (BAIDU_UNLIKELY(rc != UBRING_OK)) { LOG(ERROR) << "Connect Trx map shared memory failed, remote shm=" << remote_trx_shm.name; return rc; } - if (UNLIKELY(UbrAddTimer() != UBRING_OK)) { + if (BAIDU_UNLIKELY(UbrAddTimer() != UBRING_OK)) { LOG(ERROR) << "Ubr add timer failed, local_name=" << local_name; ShmRemoteFree(&_trx->remote_shm); return UBRING_ERR; @@ -1090,7 +1083,7 @@ RETURN_CODE UBRing::UbrMapRemoteShmAddTimer(SHM *local_trx_shm, const char *loca RETURN_CODE UBRing::ApplyAndMapLocalShm(SHM *local_trx_shm, const char *local_name) { - if (UNLIKELY(_trx == nullptr || local_trx_shm == nullptr)) { + if (BAIDU_UNLIKELY(_trx == nullptr || local_trx_shm == nullptr)) { LOG(ERROR) << "Trx map Shared memory failed, trx is null, local_name=" << local_name; return UBRING_ERR; } @@ -1100,13 +1093,13 @@ RETURN_CODE UBRing::ApplyAndMapLocalShm(SHM *local_trx_shm, const char *local_na SHM_NAME_PREFIX, local_name, CLIENT_SHM_NAME_SUFFIX); - if (UNLIKELY(result < 0)) { + if (BAIDU_UNLIKELY(result < 0)) { LOG(ERROR) << "Copy client localTrx shared memory name failed, local_name=" << local_name << ", ret=" << result; return UBRING_ERR; } RETURN_CODE rc = ShmLocalCalloc(local_trx_shm); - if (UNLIKELY(rc != UBRING_OK)) { + if (BAIDU_UNLIKELY(rc != UBRING_OK)) { LOG(ERROR) << "Trx apply local shared memory failed, local shm name=" << local_trx_shm->name << ", rc=" << rc; if (rc == SHM_ERR_EXIST || rc == SHM_ERR_NOT_FOUND) { rc = UBR_ERR_ADDR_IN_USE; @@ -1115,7 +1108,7 @@ RETURN_CODE UBRing::ApplyAndMapLocalShm(SHM *local_trx_shm, const char *local_na return rc; } rc = UbrTrxMapLocalShm(local_trx_shm); - if (UNLIKELY(rc != UBRING_OK)) { + if (BAIDU_UNLIKELY(rc != UBRING_OK)) { LOG(ERROR) << "Trx map local shared memory failed, local shm name=" << local_trx_shm->name; ShmLocalFree(local_trx_shm); UBRingManager::ReleaseUbrTrxFromMgr(_trx, ATOMIC_LOAD(_trx->ubr_id)); @@ -1137,12 +1130,12 @@ RETURN_CODE UBRing::ApplyAndMapLocalShm(SHM *local_trx_shm, const char *local_na RETURN_CODE UBRing::ApplyAndMapRemoteShm(SHM *remote_trx_shm) { RETURN_CODE rc = ShmRemoteMalloc(remote_trx_shm); - if (UNLIKELY(rc != UBRING_OK)) { + if (BAIDU_UNLIKELY(rc != UBRING_OK)) { LOG(ERROR) << "Trx apply remote shared memory failed."; return rc; } rc = UbrTrxMapRemoteShm(remote_trx_shm); - if (UNLIKELY(rc != UBRING_OK)) { + if (BAIDU_UNLIKELY(rc != UBRING_OK)) { LOG(ERROR) << "Trx map shared memory failed."; ShmRemoteFree(remote_trx_shm); return rc; @@ -1173,13 +1166,13 @@ RETURN_CODE UBRing::WritevHasEnoughSpace(size_t buf_len) RETURN_CODE UBRing::UbrClearResourceCheck(UbrTrx *trx) { - if (UNLIKELY(trx == nullptr)) { + if (BAIDU_UNLIKELY(trx == nullptr)) { LOG(ERROR) << "Trx close failed, trx is null."; return UBRING_ERR; } UbrEventQMsg* local_tx_event_q = (UbrEventQMsg *)trx->ubr_tx.local_tx_event_q.addr; - if (UNLIKELY(local_tx_event_q == nullptr)) { + if (BAIDU_UNLIKELY(local_tx_event_q == nullptr)) { LOG(ERROR) << "Trx close failed, local_tx_event_q addr is NULL, trx local name=" << trx->local_shm.name; return UBRING_ERR; } @@ -1217,7 +1210,7 @@ RETURN_CODE UBRing::ClearTrxResource(UbrTrx *trx) RETURN_CODE UBRing::UbrTrxCloseCheck(UbrTrx *trx) { - if (UNLIKELY(trx == nullptr)) { + if (BAIDU_UNLIKELY(trx == nullptr)) { LOG(ERROR) << "Trx close failed, client trx is null."; return UBRING_ERR; } @@ -1227,7 +1220,7 @@ RETURN_CODE UBRing::UbrTrxCloseCheck(UbrTrx *trx) return UBRING_REENTRY; } - if (UNLIKELY(trx->ubr_tx.local_tx_event_q.addr == nullptr)) { + if (BAIDU_UNLIKELY(trx->ubr_tx.local_tx_event_q.addr == nullptr)) { LOG(ERROR) << "Trx close failed, local_tx_event_q addr is NULL, trx local name=" << trx->local_shm.name; return UBRING_ERR; } @@ -1242,7 +1235,7 @@ ssize_t UBRing::StartReadv(UbrTrx *trx, const struct iovec *iov, int iovcnt, siz UbrMsgFormat *data_msg = (UbrMsgFormat *)trx->ubr_rx.local_data_q.addr; bool not_eof_encountered = true; while (not_eof_encountered && remain_buf_len > 0) { - if (UNLIKELY(CheckTrxRecvPreCheck(trx) != UBRING_OK)) { + if (BAIDU_UNLIKELY(CheckTrxRecvPreCheck(trx) != UBRING_OK)) { return UBRING_ERR; } UbrMsgFormat *current_chunk = &data_msg[trx->ubr_rx.read_pos]; @@ -1259,7 +1252,7 @@ ssize_t UBRing::StartReadv(UbrTrx *trx, const struct iovec *iov, int iovcnt, siz } uint8_t chunk_msg_len = current_chunk->header[UBR_MSG_LEN_INDEX]; uint8_t cur_index = current_chunk->header[UBR_MSG_CUR_INDEX]; - if (UNLIKELY(!IsRecvChunkHeaderValid(chunk_msg_len, cur_index))) { + if (BAIDU_UNLIKELY(!IsRecvChunkHeaderValid(chunk_msg_len, cur_index))) { LOG(ERROR) << "Trx readv failed, invalid chunk header msg_len=" << (uint32_t)chunk_msg_len << " cur_index=" << (uint32_t)cur_index; errno = EBADMSG; diff --git a/src/brpc/ubshm/ub_ring.h b/src/brpc/ubshm/ub_ring.h index 0e47a04d4e..16dd59e21e 100644 --- a/src/brpc/ubshm/ub_ring.h +++ b/src/brpc/ubshm/ub_ring.h @@ -98,11 +98,11 @@ class UBRing : public butil::IReader { static inline RETURN_CODE CheckTrxConnectParam(const char *listener_name, const char *local_name) { - if (UNLIKELY(listener_name == nullptr)) { + if (BAIDU_UNLIKELY(listener_name == nullptr)) { LOG(ERROR) << "The request listener name is null."; return UBRING_ERR; } - if (UNLIKELY(local_name == nullptr)) { + if (BAIDU_UNLIKELY(local_name == nullptr)) { LOG(ERROR) << "The request trx shared memory name is null."; return UBRING_ERR; } @@ -119,7 +119,7 @@ class UBRing : public butil::IReader { static inline RETURN_CODE CheckTrxSendPreCheck(UbrTrx *trx) { - if (UNLIKELY(trx->ubr_tx.trx_state != UBR_STATE_CONNECTED)) { + if (BAIDU_UNLIKELY(trx->ubr_tx.trx_state != UBR_STATE_CONNECTED)) { LOG(ERROR) << "Trx send failed, trx is not connected state."; return UBRING_ERR; } @@ -128,25 +128,25 @@ class UBRing : public butil::IReader { } static RETURN_CODE CheckTrxRecvParam(UbrTrx *trx, const void *buf, uint32_t buf_len) { - if (UNLIKELY(trx == nullptr)) { + if (BAIDU_UNLIKELY(trx == nullptr)) { LOG(ERROR) << "Trx recv failed, trx is null."; return UBRING_ERR; } - if (UNLIKELY((UbrEventQMsg *)trx->ubr_rx.local_rx_event_q.addr == nullptr)) { + if (BAIDU_UNLIKELY((UbrEventQMsg *)trx->ubr_rx.local_rx_event_q.addr == nullptr)) { LOG(ERROR) << "Trx send failed, local_tx_event_q addr is NULL."; return UBRING_ERR; } - if (UNLIKELY(trx->ubr_rx.trx_state != UBR_STATE_CONNECTED)) { + if (BAIDU_UNLIKELY(trx->ubr_rx.trx_state != UBR_STATE_CONNECTED)) { LOG(ERROR) << "Trx recv failed, trx is not connected statep=" << trx->ubr_rx.trx_state; return UBR_NOT_CONNECTED; } - if (UNLIKELY(buf == nullptr)) { + if (BAIDU_UNLIKELY(buf == nullptr)) { LOG(ERROR) << "Trx recv failed, buf is null."; return UBRING_ERR; } - if (UNLIKELY(buf_len == 0)) { + if (BAIDU_UNLIKELY(buf_len == 0)) { LOG(ERROR) << "Trx recv failed, buf_len is 0."; return UBRING_ERR; } @@ -155,7 +155,7 @@ class UBRing : public butil::IReader { static inline RETURN_CODE CheckTrxRecvPreCheck(UbrTrx *trx) { - if (UNLIKELY(trx->ubr_rx.trx_state != UBR_STATE_CONNECTED)) { + if (BAIDU_UNLIKELY(trx->ubr_rx.trx_state != UBR_STATE_CONNECTED)) { LOG(ERROR) << "Trx recv failed, trx is not connected state."; return UBRING_ERR; } @@ -188,15 +188,15 @@ class UBRing : public butil::IReader { LOG(ERROR) << "Trx close callback failed, trx is null."; return UBRING_ERR; } - if (UNLIKELY(trx->local_shm.addr == nullptr)) { + if (BAIDU_UNLIKELY(trx->local_shm.addr == nullptr)) { LOG(ERROR) << "Trx close failed, local_shm addr is NULL."; return UBRING_ERR; } - if (UNLIKELY(trx->ubr_rx.local_rx_event_q.addr == nullptr)) { + if (BAIDU_UNLIKELY(trx->ubr_rx.local_rx_event_q.addr == nullptr)) { LOG(ERROR) << "Trx close failed, local_rx_event_q addr is NULL."; return UBRING_ERR; } - if (UNLIKELY(trx->ubr_tx.local_tx_event_q.addr == nullptr)) { + if (BAIDU_UNLIKELY(trx->ubr_tx.local_tx_event_q.addr == nullptr)) { LOG(ERROR) << "Trx close failed, local_tx_event_q addr is NULL."; return UBRING_ERR; } diff --git a/src/brpc/ubshm/ub_ring_manager.cpp b/src/brpc/ubshm/ub_ring_manager.cpp index 6abb485e8a..9084596bba 100644 --- a/src/brpc/ubshm/ub_ring_manager.cpp +++ b/src/brpc/ubshm/ub_ring_manager.cpp @@ -15,10 +15,12 @@ // specific language governing permissions and limitations // under the License. +#include #include #include "brpc/ubshm/ub_ring.h" #include "brpc/ubshm/ub_ring_manager.h" #include "butil/logging.h" +#include "butil/scoped_lock.h" namespace brpc { namespace ubring { @@ -41,11 +43,11 @@ uint64_t g_ub_event_cnt = 0; uint64_t g_ubr_listener_num = 0; RETURN_CODE UBRingManager::GetUbrDealMsgMaxCnt(const uint32_t capacity, uint32_t *deal_msg_max_cnt) { - if (UNLIKELY(deal_msg_max_cnt == nullptr)) { + if (BAIDU_UNLIKELY(deal_msg_max_cnt == nullptr)) { LOG(ERROR) << "Get update factor failed, deal_msg_max_cnt is null."; return UBRING_ERR; } - if (UNLIKELY(FLAGS_tail_update_after_read == 0)) { + if (BAIDU_UNLIKELY(FLAGS_tail_update_after_read == 0)) { LOG(ERROR) << "Get update factor failed, factor is 0."; return UBRING_ERR; } @@ -66,7 +68,7 @@ RETURN_CODE UBRingManager::UbrMgrDefault() RETURN_CODE UBRingManager::UbrMgrInit() { RETURN_CODE rc = UbrMgrDefault(); - if (UNLIKELY(rc != UBRING_OK)) { + if (BAIDU_UNLIKELY(rc != UBRING_OK)) { LOG(ERROR) << "Ubr manager set default values failed."; return rc; } @@ -79,7 +81,7 @@ RETURN_CODE UBRingManager::UbrMgrInit() { g_ubr_mgr.trx_mgr_unit_id = (uint64_t *)malloc(trx_mgr_id_size); size_t trx_mgr_ctl_size = g_ubr_mgr.trx_cap * sizeof(UbrCleanupCtl *); g_ubr_mgr.trx_mgr_unit_ctl = (UbrCleanupCtl **)malloc(trx_mgr_ctl_size); - if (UNLIKELY(g_ubr_mgr.trx_mgr == nullptr || + if (BAIDU_UNLIKELY(g_ubr_mgr.trx_mgr == nullptr || g_ubr_mgr.trx_mgr_unit_status == nullptr || g_ubr_mgr.trx_mgr_unit_id == nullptr || g_ubr_mgr.trx_mgr_unit_ctl == nullptr)) { @@ -105,7 +107,7 @@ void UBRingManager::UbrMgrFini() { while (busy) { busy = false; { - LOCK_GUARD(g_ubr_trx_mgr_mtx); + BAIDU_SCOPED_LOCK(g_ubr_trx_mgr_mtx); if (g_ubr_mgr.trx_mgr_unit_ctl != nullptr) { for (uint32_t i = 0; i < g_ubr_mgr.trx_cap; ++i) { UbrCleanupCtl* ctl = g_ubr_mgr.trx_mgr_unit_ctl[i]; @@ -127,7 +129,7 @@ void UBRingManager::UbrMgrFini() { } } { - LOCK_GUARD(g_ubr_trx_mgr_mtx); + BAIDU_SCOPED_LOCK(g_ubr_trx_mgr_mtx); if (g_ubr_mgr.trx_mgr_unit_ctl != nullptr) { for (uint32_t i = 0; i < g_ubr_mgr.trx_cap; ++i) { UbrCleanupCtl* ctl = g_ubr_mgr.trx_mgr_unit_ctl[i]; @@ -143,7 +145,7 @@ void UBRingManager::UbrMgrFini() { FREE_PTR(g_ubr_mgr.trx_mgr_unit_ctl); } { - LOCK_GUARD(g_ubr_listener_mgr_mtx); + BAIDU_SCOPED_LOCK(g_ubr_listener_mgr_mtx); } g_ubr_mgr.trx_num = 0; g_ubr_mgr.trx_cap = 0; @@ -151,17 +153,17 @@ void UBRingManager::UbrMgrFini() { } RETURN_CODE UBRingManager::AcquireUbrTrxFromMgr(UbrTrx **trx) { - if (UNLIKELY(trx == nullptr)) { + if (BAIDU_UNLIKELY(trx == nullptr)) { LOG(ERROR) << "Acquire trx failed, trx is null."; return UBRING_ERR; } - if (UNLIKELY(g_ubr_mgr.trx_mgr == nullptr)) { + if (BAIDU_UNLIKELY(g_ubr_mgr.trx_mgr == nullptr)) { LOG(ERROR) << "Acquire trx failed, trx_mgr is null."; return UBRING_ERR; } - LOCK_GUARD(g_ubr_trx_mgr_mtx); + BAIDU_SCOPED_LOCK(g_ubr_trx_mgr_mtx); if (g_ubr_mgr.trx_num >= g_ubr_mgr.trx_cap) { LOG(ERROR) << "Acquire trx failed, trx number is full."; return UBRING_ERR; @@ -169,9 +171,12 @@ RETURN_CODE UBRingManager::AcquireUbrTrxFromMgr(UbrTrx **trx) { for (uint32_t i = 0; i < g_ubr_mgr.trx_cap; ++i) { if (g_ubr_mgr.trx_mgr_unit_status[i] == UBR_MGR_UNIT_FREE) { - memset(&g_ubr_mgr.trx_mgr[i], 0, sizeof(UbrTrx)); - // The explicit re-initialization after memset is deliberate: it - // documents the per-acquisition invariants of these fields. + // Value-initialize the slot so its butil::atomic members are + // properly constructed while every other field is zeroed, which + // is what the previous memset did. + new (&g_ubr_mgr.trx_mgr[i]) UbrTrx(); + // The explicit re-initialization is deliberate: it documents the + // per-acquisition invariants of these fields. g_ubr_mgr.trx_mgr[i].close_timer = nullptr; g_ubr_mgr.trx_mgr[i].hb_timer = nullptr; g_ubr_mgr.trx_mgr[i].cleanup_ctl = nullptr; @@ -202,23 +207,23 @@ RETURN_CODE UBRingManager::AcquireUbrTrxFromMgr(UbrTrx **trx) { RETURN_CODE UBRingManager::ReleaseUbrTrxFromMgr(UbrTrx *trx, uint64_t expect_ubr_id) { - if (UNLIKELY(trx == nullptr)) { + if (BAIDU_UNLIKELY(trx == nullptr)) { LOG(ERROR) << "Release trx failed, trx is null."; return UBRING_ERR; } - if (UNLIKELY(g_ubr_mgr.trx_mgr == nullptr)) { + if (BAIDU_UNLIKELY(g_ubr_mgr.trx_mgr == nullptr)) { LOG(ERROR) << "Release trx failed, trx_mgr is null."; return UBRING_ERR; } - LOCK_GUARD(g_ubr_trx_mgr_mtx); + BAIDU_SCOPED_LOCK(g_ubr_trx_mgr_mtx); uint32_t idx = trx->trx_mgr_index; if (g_ubr_mgr.trx_mgr_unit_status[idx] == UBR_MGR_UNIT_FREE) { LOG(INFO) << "Release trx already freed, name=" << trx->local_shm.name; return UBRING_OK; } - if (UNLIKELY(g_ubr_mgr.trx_mgr_unit_id[idx] != expect_ubr_id)) { + if (BAIDU_UNLIKELY(g_ubr_mgr.trx_mgr_unit_id[idx] != expect_ubr_id)) { // The slot was released and acquired again meanwhile; the stale // caller must not touch the new occupant. LOG(WARNING) << "Release stale trx refused, name=" << trx->local_shm.name; @@ -242,8 +247,8 @@ RETURN_CODE UBRingManager::ReleaseUbrTrxFromMgr(UbrTrx *trx, } UbrCleanupCtl* UBRingManager::SnapshotUnitCleanupCtl(uint32_t idx) { - LOCK_GUARD(g_ubr_trx_mgr_mtx); - if (UNLIKELY(g_ubr_mgr.trx_mgr_unit_ctl == nullptr || idx >= g_ubr_mgr.trx_cap)) { + BAIDU_SCOPED_LOCK(g_ubr_trx_mgr_mtx); + if (BAIDU_UNLIKELY(g_ubr_mgr.trx_mgr_unit_ctl == nullptr || idx >= g_ubr_mgr.trx_cap)) { return nullptr; } UbrCleanupCtl* ctl = g_ubr_mgr.trx_mgr_unit_ctl[idx]; @@ -254,8 +259,8 @@ UbrCleanupCtl* UBRingManager::SnapshotUnitCleanupCtl(uint32_t idx) { } bool UBRingManager::IsUbrTrxSlotUsed(uint32_t idx, uint64_t expect_ubr_id) { - LOCK_GUARD(g_ubr_trx_mgr_mtx); - if (UNLIKELY(g_ubr_mgr.trx_mgr_unit_id == nullptr || + BAIDU_SCOPED_LOCK(g_ubr_trx_mgr_mtx); + if (BAIDU_UNLIKELY(g_ubr_mgr.trx_mgr_unit_id == nullptr || g_ubr_mgr.trx_mgr_unit_status == nullptr || idx >= g_ubr_mgr.trx_cap)) { return false; @@ -267,8 +272,8 @@ bool UBRingManager::IsUbrTrxSlotUsed(uint32_t idx, uint64_t expect_ubr_id) { bool UBRingManager::TryPublishUnitCleanupCtl(uint32_t idx, uint64_t expect_ubr_id, UbrCleanupCtl *ctl) { - LOCK_GUARD(g_ubr_trx_mgr_mtx); - if (UNLIKELY(g_ubr_mgr.trx_mgr_unit_ctl == nullptr || + BAIDU_SCOPED_LOCK(g_ubr_trx_mgr_mtx); + if (BAIDU_UNLIKELY(g_ubr_mgr.trx_mgr_unit_ctl == nullptr || g_ubr_mgr.trx_mgr_unit_status == nullptr || g_ubr_mgr.trx_mgr_unit_id == nullptr || idx >= g_ubr_mgr.trx_cap || @@ -283,8 +288,8 @@ bool UBRingManager::TryPublishUnitCleanupCtl(uint32_t idx, } bool UBRingManager::DetachUnitCleanupCtl(uint32_t idx, UbrCleanupCtl *ctl) { - LOCK_GUARD(g_ubr_trx_mgr_mtx); - if (UNLIKELY(g_ubr_mgr.trx_mgr_unit_ctl == nullptr || + BAIDU_SCOPED_LOCK(g_ubr_trx_mgr_mtx); + if (BAIDU_UNLIKELY(g_ubr_mgr.trx_mgr_unit_ctl == nullptr || idx >= g_ubr_mgr.trx_cap || g_ubr_mgr.trx_mgr_unit_ctl[idx] != ctl)) { return false; @@ -320,7 +325,7 @@ void UBRingManager::LinkInfoFini(void) { return; } { - LOCK_GUARD(g_link_info_mgr_mtx); + BAIDU_SCOPED_LOCK(g_link_info_mgr_mtx); FREE_PTR(g_link_info_mgr.all_link_info); FREE_PTR(g_link_info_mgr.link_mgr_unit_status); } @@ -366,11 +371,11 @@ void UBRingManager::ReleaseLinkInfoFromMgr(UbrTrx *trx) { int32_t UBRingManager::UbEventCallback(const char *shm_name) { - if (UNLIKELY(shm_name == nullptr)) { + if (BAIDU_UNLIKELY(shm_name == nullptr)) { LOG(ERROR) << "Ub event callback failed, shm name is null."; return UBRING_ERR; } - if (UNLIKELY(g_ubr_mgr.trx_mgr == nullptr)) { + if (BAIDU_UNLIKELY(g_ubr_mgr.trx_mgr == nullptr)) { LOG(ERROR) << "Ub event callback failed, trx mgr is null."; return UBRING_ERR; } diff --git a/src/brpc/ubshm/ubr_trx.h b/src/brpc/ubshm/ubr_trx.h index 6a947d2d78..cc35f7223f 100644 --- a/src/brpc/ubshm/ubr_trx.h +++ b/src/brpc/ubshm/ubr_trx.h @@ -144,7 +144,7 @@ struct UbrCleanupCtl { TagUbrTrx* trx; // immutable uint64_t ubr_id; // immutable generation AtomicInt state; // UbrCleanupState - UbrTimerId timer; // delayed clear timer + butil::atomic timer; // delayed clear timer // References: timer/callback (held from a successful schedule until // the callback fully returned, or released by whoever cancels the // timer before it fires) + starter (until the schedule call @@ -167,9 +167,9 @@ typedef struct TagUbrTrx { UbrTrxType type; SHM local_shm; SHM remote_shm; - UbrTimerId close_timer; - UbrTimerId hb_timer; - UbrCleanupCtl* cleanup_ctl; + butil::atomic close_timer; + butil::atomic hb_timer; + butil::atomic cleanup_ctl; // Last io ids seen by the close-check timer, used to reset its // back-off polling interval when the link has traffic. uint64_t close_chk_in_io_id; From d3aa6588822b146d24ad7a3ae32db287c90e60a3 Mon Sep 17 00:00:00 2001 From: Sherlock0203 <1297399478@qq.com> Date: Tue, 29 Sep 2026 15:55:03 +0800 Subject: [PATCH 4/9] fix(ubshm): validate timer generations and quiesce callbacks on teardown Address the ubring lifecycle review on the bthread-timer refactor. Generation validation: - UbrTimerStart/UbrTimerTask carry an opaque `gen' and pass it back to the callback on every fire, so a callback armed for an object that was later released and reused can detect that it is stale. `gen' is required and sits before the optional backoff; the UBS shm timer passes 0 explicitly. - UbrScheduleClearTimer takes expect_ubr_id, rejects a mismatch before creating the cleanup ctl, and stamps ctl->ubr_id with it instead of reading the slot's current generation. - UbrTrxCallbackCheck runs before the generation check in the close and heartbeat callbacks, so trx is null-checked before trx->ubr_id is dereferenced; UbrTrxClose and UbEventCallback pass the generation they observed for the slot. - UbrPassiveClearTrx/ClearTrxResource/UbrAddAsynClearTimer thread expect_ubr_id through. Teardown: - ReleaseUbrTrxFromMgr disarms the per-acquisition close/heartbeat timers after the generation check and before the slot is marked FREE, so a rollback no longer orphans a periodic task onto the slot's next occupant. - UbrMgrFini snapshots the USED slots under g_ubr_trx_mgr_mtx and UbrTimerDelAndWait's their timers outside the lock before the pool is freed; the previous loop only waited for cleanup-ctl callbacks, so a dispatched close/heartbeat callback could still be reading UbrTrx when FREE_PTR(trx_mgr) ran. The walk is guarded by "all four pool arrays are allocated", which also stops the ctl loops from reading arrays that UbrMgrInit's allocation-failure path leaves uninitialized. - UbrClearResourceCheck/UbrPassiveClearTrx stop the per-trx timers through UbrStopTrxTimer, which waits with UbrTimerDelAndWait unless the caller is the timer callback itself (in_timer_callback), so teardown from a user thread or from the ubsmem fault callback cannot clear a trx whose callback is mid-flight. UbrTrxCloseCallback re-validates its event queue pointers after the generation check, like UbrTrxHBCallback, so a concurrent release cannot turn the dereference into a null access. The ordering argument is a synchronization one: bthread dispatches every timer callback from a single global timer thread, so two per-trx callbacks never run concurrently and the one-shot delayed-clear callback is ordered after them; ub_flying_io_timeout_s only delays the clear so in-flight IO can drain. Non-blocking UbrTimerDel is kept on the paths that hold g_ubr_trx_mgr_mtx (ReleaseUbrTrxFromMgr and the callback side), where the callbacks take the same lock and a wait would deadlock. Tested: cmake --build cmake-build-debug --target brpc_ubring_unittest, and ./test/brpc_ubring_unittest (12/12 passed). --- src/brpc/ubshm/shm/shm_ubs.cpp | 4 +- src/brpc/ubshm/shm/shm_ubs.h | 3 +- src/brpc/ubshm/timer/timer_mgr.cpp | 19 +++-- src/brpc/ubshm/timer/timer_mgr.h | 11 ++- src/brpc/ubshm/ub_ring.cpp | 129 +++++++++++++++++++++++------ src/brpc/ubshm/ub_ring.h | 23 +++-- src/brpc/ubshm/ub_ring_manager.cpp | 51 +++++++++++- 7 files changed, 187 insertions(+), 53 deletions(-) diff --git a/src/brpc/ubshm/shm/shm_ubs.cpp b/src/brpc/ubshm/shm/shm_ubs.cpp index 89f6702a6b..193ba52a96 100644 --- a/src/brpc/ubshm/shm/shm_ubs.cpp +++ b/src/brpc/ubshm/shm/shm_ubs.cpp @@ -411,7 +411,7 @@ static void DeleteShmToList(ShmList* shm_list) shm_list->size--; } -void *UbsShmCallback(void* args) +void *UbsShmCallback(void* args, uint64_t) { ShmList *shm_list = (ShmList*)args; if (BAIDU_UNLIKELY(shm_list == nullptr)) { @@ -461,7 +461,7 @@ RETURN_CODE UbsShmAddTimer(ShmList *shm_list) { const uint64_t timer_interval_us = (uint64_t)FLAGS_ub_flying_io_timeout_s * SEC_TO_USEC; RETURN_CODE rc = UbrTimerStart(&g_shm_timer_id, 0, timer_interval_us, - UbsShmCallback, (void*)shm_list); + UbsShmCallback, (void*)shm_list, 0); if (BAIDU_UNLIKELY(rc != UBRING_OK)) { LOG(ERROR) << "Start shm timer failed."; return UBRING_ERR; diff --git a/src/brpc/ubshm/shm/shm_ubs.h b/src/brpc/ubshm/shm/shm_ubs.h index ee936c0a55..39f622cf76 100644 --- a/src/brpc/ubshm/shm/shm_ubs.h +++ b/src/brpc/ubshm/shm/shm_ubs.h @@ -17,6 +17,7 @@ #ifndef BRPC_SHM_UBS_H #define BRPC_SHM_UBS_H +#include namespace brpc { namespace ubring { DECLARE_int32(ub_flying_io_timeout_s); @@ -40,7 +41,7 @@ RETURN_CODE UbsShmFini(void); RETURN_CODE UbsShmLocalMmap(SHM *shm, int prot); void UbsMemLoggerPrint(int level, const char *msg); -void *UbsShmCallback(void* args); +void *UbsShmCallback(void* args, uint64_t gen); RETURN_CODE UbsShmAddTimer(ShmList *shm_list); RETURN_CODE InitShmTimer(ShmList **shm_list); RETURN_CODE DestroyShmTimer(ShmList *shm_list); diff --git a/src/brpc/ubshm/timer/timer_mgr.cpp b/src/brpc/ubshm/timer/timer_mgr.cpp index 912e331c1e..28a5d74f7c 100644 --- a/src/brpc/ubshm/timer/timer_mgr.cpp +++ b/src/brpc/ubshm/timer/timer_mgr.cpp @@ -48,8 +48,9 @@ enum UbrTimerState { struct UbrTimerTask { butil::atomic* slot; butil::atomic id; - void* (*cb)(void*); + void* (*cb)(void*, uint64_t); void* arg; + uint64_t gen; // opaque, passed back to cb UbrTimerBackoffFn backoff; uint64_t interval_us; // timer thread only bool periodic; @@ -77,7 +78,7 @@ void UbrTimerOnFire(void* p) { if (task->periodic) { if (!task->stopped.load()) { - task->cb(task->arg); + task->cb(task->arg, task->gen); } // Claim the next schedule's ref before re-reading `stopped' so a // racing delete can neither free the task nor orphan a re-arm. @@ -119,7 +120,7 @@ void UbrTimerOnFire(void* p) { UbrTimerId expected = task; const bool owned = task->slot->compare_exchange_strong(expected, nullptr); if (owned) { - task->cb(task->arg); + task->cb(task->arg, task->gen); } ReleaseRef(task); // schedule if (owned) { @@ -132,8 +133,9 @@ UbrTimerTask* TakeOutTask(butil::atomic* slot) { } RETURN_CODE TimerStartInternal(butil::atomic* slot, uint64_t delay_us, - uint64_t interval_us, void* (*cb)(void*), - void* arg, UbrTimerBackoffFn backoff) { + uint64_t interval_us, void* (*cb)(void*, uint64_t), + void* arg, uint64_t gen, + UbrTimerBackoffFn backoff) { if (BAIDU_UNLIKELY(slot == nullptr || cb == nullptr)) { LOG(ERROR) << "Ubr timer start invalid argument, slot=" << slot; return UBRING_ERR; @@ -148,6 +150,7 @@ RETURN_CODE TimerStartInternal(butil::atomic* slot, uint64_t delay_u task->id.store(0); task->cb = cb; task->arg = arg; + task->gen = gen; task->backoff = backoff; task->interval_us = interval_us; task->periodic = (interval_us > 0); @@ -196,9 +199,9 @@ RETURN_CODE TimerStartInternal(butil::atomic* slot, uint64_t delay_u } // namespace RETURN_CODE UbrTimerStart(butil::atomic* slot, uint64_t delay_us, - uint64_t interval_us, void* (*cb)(void*), - void* arg, UbrTimerBackoffFn backoff) { - return TimerStartInternal(slot, delay_us, interval_us, cb, arg, backoff); + uint64_t interval_us, void* (*cb)(void*, uint64_t), + void* arg, uint64_t gen, UbrTimerBackoffFn backoff) { + return TimerStartInternal(slot, delay_us, interval_us, cb, arg, gen, backoff); } int UbrTimerDel(butil::atomic* slot) { diff --git a/src/brpc/ubshm/timer/timer_mgr.h b/src/brpc/ubshm/timer/timer_mgr.h index 24ef2e0b15..04b43c02b0 100644 --- a/src/brpc/ubshm/timer/timer_mgr.h +++ b/src/brpc/ubshm/timer/timer_mgr.h @@ -35,16 +35,19 @@ typedef struct UbrTimerTask* UbrTimerId; // Runs on the timer thread only. typedef uint64_t (*UbrTimerBackoffFn)(void* arg, uint64_t cur_interval_us); -// Schedule `cb(arg)' to run after `delay_us' and, when `interval_us' > 0, +// Schedule `cb(arg, gen)' to run after `delay_us' and, when `interval_us' > 0, // re-arm itself after every run until deleted. One-shot timers release // their handle slot before running the callback, so the callback may free // the object that stores the slot; the task object itself is released // automatically. `slot' must point to a real butil::atomic // object, so start and delete can race on one RMW without reinterpreting -// plain storage as an atomic. +// plain storage as an atomic. `gen' is an opaque value carried by the task +// and passed back on every fire, so a callback armed for an object that was +// later released and reused can detect that it is stale. It is required: +// passing a wrong value silently disables the generation guard. RETURN_CODE UbrTimerStart(butil::atomic* slot, uint64_t delay_us, - uint64_t interval_us, void* (*cb)(void*), - void* arg, + uint64_t interval_us, void* (*cb)(void*, uint64_t), + void* arg, uint64_t gen, UbrTimerBackoffFn backoff = nullptr); // Non-blocking delete, safe to call from inside the timer callback itself. diff --git a/src/brpc/ubshm/ub_ring.cpp b/src/brpc/ubshm/ub_ring.cpp index 3ba1af8aa8..1d9b6bbc36 100644 --- a/src/brpc/ubshm/ub_ring.cpp +++ b/src/brpc/ubshm/ub_ring.cpp @@ -68,6 +68,26 @@ RETURN_CODE UBRing::UbrTrxMapShm(SHM *local_shm, SHM *remote_shm) return UBRING_OK; } +// Stop a per-trx timer before its slot becomes reusable. Outside a per-trx +// timer callback this waits -- UbrTimerDelAndWait returns only after a +// callback that was already dispatched has left the trx -- which is what lets +// the caller clear the trx and free its shared memory afterwards. Inside the +// callback itself the wait would join the task that is currently running, so +// the non-blocking delete is used instead; no wait is needed there, because +// bthread dispatches every timer callback from one global timer thread +// (TimerThread in bthread/timer_thread.{h,cpp} is created by a single +// pthread_once), so this trx's sibling timer callback cannot be running +// concurrently and the non-blocking delete already removed it from the timer +// heap. +static void UbrStopTrxTimer(butil::atomic* slot, + bool in_timer_callback) { + if (in_timer_callback) { + UbrTimerDel(slot); + } else { + UbrTimerDelAndWait(slot); + } +} + static void UbrDoAsynClearWork(UbrTrx *trx, uint64_t expect_ubr_id) { if (BAIDU_UNLIKELY(UBRing::UbrTrxFreeShm(trx) != UBRING_OK)) { LOG(ERROR) << "Trx close, wait for local shm " << trx->local_shm.name << " free fail."; @@ -97,11 +117,17 @@ static void UbrDoPassiveClearWork(UbrTrx *trx, uint64_t expect_ubr_id) { // per-acquisition control object, so exactly one of the delayed-clear // callback and a force close ever runs the cleanup. `work' is the cleanup // body, used directly when the timer cannot be started. -static RETURN_CODE UbrScheduleClearTimer(UbrTrx *trx, void* (*cb)(void*), +static RETURN_CODE UbrScheduleClearTimer(UbrTrx *trx, uint64_t expect_ubr_id, + void* (*cb)(void*, uint64_t), void (*work)(UbrTrx*, uint64_t)) { if (BAIDU_UNLIKELY(trx == nullptr || trx->local_shm.addr == nullptr)) { return UBRING_OK; // released trx, stale event } + // A callback that outlived its generation must not capture the id of the + // slot's new occupant, nor schedule cleanup for that new transaction. + if (BAIDU_UNLIKELY(ATOMIC_LOAD(trx->ubr_id) != expect_ubr_id)) { + return UBRING_OK; // stale event on a reused slot + } if (trx->cleanup_ctl.load() != nullptr) { return UBRING_OK; // cleanup already scheduled } @@ -111,7 +137,7 @@ static RETURN_CODE UbrScheduleClearTimer(UbrTrx *trx, void* (*cb)(void*), return UBRING_ERR; } ctl->trx = trx; - ctl->ubr_id = ATOMIC_LOAD(trx->ubr_id); + ctl->ubr_id = expect_ubr_id; ctl->state.store(UBR_CLEANUP_PENDING); ctl->timer = nullptr; ctl->ref.store(2); // timer/callback + starter; the @@ -135,7 +161,8 @@ static RETURN_CODE UbrScheduleClearTimer(UbrTrx *trx, void* (*cb)(void*), return UBRING_OK; } RETURN_CODE rc = UbrTimerStart(&ctl->timer, - (uint64_t)FLAGS_ub_flying_io_timeout_s * SEC_TO_USEC, 0, cb, ctl); + (uint64_t)FLAGS_ub_flying_io_timeout_s * SEC_TO_USEC, 0, cb, ctl, + expect_ubr_id); if (BAIDU_UNLIKELY(rc != UBRING_OK)) { // The timer was never scheduled: this path owns the manager, // timer/callback and starter references. Roll the schedule back @@ -186,6 +213,7 @@ RETURN_CODE UBRing::UbrTrxClose() { } return UBRING_ERR; } + const uint64_t expect_ubr_id = ATOMIC_LOAD(_trx->ubr_id); if (_trx->ubr_rx.remote_tx_event_q.addr != nullptr) { ((UbrEventQMsg *)_trx->ubr_rx.remote_tx_event_q.addr)->flag = UBR_STATE_CLOSING; } @@ -211,7 +239,6 @@ RETURN_CODE UBRing::UbrTrxClose() { // delayed cleanup, then settle the cleanup ownership: force // runs the cleanup itself when it can claim it, and leaves it // to an already running delayed-clear callback otherwise. - const uint64_t expect_ubr_id = ATOMIC_LOAD(_trx->ubr_id); UbrTimerDelAndWait(&_trx->close_timer); UbrTimerDelAndWait(&_trx->hb_timer); UbrCleanupCtl* ctl = UBRingManager::SnapshotUnitCleanupCtl(_trx->trx_mgr_index); @@ -254,7 +281,7 @@ RETURN_CODE UBRing::UbrTrxClose() { } _trx->ubr_rx.trx_state = UBR_STATE_CLOSED; RETURN_CODE rc; - if (BAIDU_UNLIKELY((rc = ClearTrxResource(_trx)) != UBRING_OK)) { + if (BAIDU_UNLIKELY((rc = ClearTrxResource(_trx, expect_ubr_id, false)) != UBRING_OK)) { if (rc == UBRING_REENTRY) { LOG(INFO) << "Trx close, peer is closing, trx local name=" << _trx->local_shm.name; return UBRING_OK; @@ -311,6 +338,7 @@ RETURN_CODE UBRing::UbrAddCloseTimer() { _trx->close_chk_out_io_id = ATOMIC_LOAD(_trx->ubr_tx.out_io_id); RETURN_CODE rc = UbrTimerStart(&_trx->close_timer, 0, interval_us, UbrTrxCloseCallback, (void*)_trx, + ATOMIC_LOAD(_trx->ubr_id), UbrCloseTimerBackoff); if (BAIDU_UNLIKELY(rc != UBRING_OK)) { LOG(ERROR) << "Start ubr close timer failed, trx local name=" << _trx->local_shm.name; @@ -333,14 +361,49 @@ RETURN_CODE UBRing::UbrAddTimer() { return UBRING_OK; } -void* UBRing::UbrTrxCloseCallback(void* args) { +// Per-trx timer callbacks run on the process-wide bthread timer thread and can +// still be dispatched after the UBRing that armed them started tearing down. +// Every access to the pooled UbrTrx is therefore gated on the generation the +// timer was armed with (the `gen' argument): the callbacks reject a stale fire +// here, and every scheduling path re-checks expect_ubr_id under +// g_ubr_trx_mgr_mtx (TryPublishUnitCleanupCtl) and again before running the +// cleanup work (IsUbrTrxSlotUsed). +// +// A slot cannot be released and reused underneath a callback that already +// passed that check, and the guarantee is a synchronization one, not a timing +// one. bthread dispatches all timer callbacks from a single global timer +// thread, so two per-trx callbacks never run concurrently, and a cleanup that +// runs from a one-shot timer callback (the delayed-clear path) is serialized +// after every per-trx callback that was dispatched before it. Teardown that +// happens on any other thread waits for a dispatched callback through +// UbrStopTrxTimer -> UbrTimerDelAndWait before it clears the trx or frees its +// shared memory (UbrClearResourceCheck, UbrPassiveClearTrx), and the force +// close and UbrMgrFini paths do the same. FLAGS_ub_flying_io_timeout_s only +// delays the delayed-clear work so that in-flight IO can drain; the callback +// side is already ordered by the timer thread. The UbrTrx pool itself is freed +// only by UbrMgrFini, which waits for these timers first. +void* UBRing::UbrTrxCloseCallback(void* args, uint64_t gen) { auto* trx = (UbrTrx*) args; + // UbrTrxCallbackCheck rejects a null trx (and cleared queues) before we + // dereference trx->ubr_id for the stale-generation check below. if (BAIDU_UNLIKELY(UBRing::UbrTrxCallbackCheck(trx) != UBRING_OK)) { return nullptr; } + // Reject a stale fire whose trx slot was released and reused: it must not + // touch the slot's new occupant. + if (BAIDU_UNLIKELY(ATOMIC_LOAD(trx->ubr_id) != gen)) { + return nullptr; + } auto* local_rx_event_q = (UbrEventQMsg *)trx->ubr_rx.local_rx_event_q.addr; auto* local_tx_event_q = (UbrEventQMsg *)trx->ubr_tx.local_tx_event_q.addr; + // UbrTrxCallbackCheck validated these before the generation check; repeat + // it here because a concurrent release may have cleared them since, and the + // dereference below is unconditional. UbrTrxHBCallback carries the same + // guard. + if (BAIDU_UNLIKELY(local_rx_event_q == nullptr || local_tx_event_q == nullptr)) { + return nullptr; + } if (local_rx_event_q->flag != UBR_STATE_CLOSED || local_tx_event_q->flag == UBR_STATE_CLOSED) { return nullptr; } @@ -361,7 +424,7 @@ void* UBRing::UbrTrxCloseCallback(void* args) { break; } remote_rx_event_q->flag = UBR_STATE_CLOSED; - RETURN_CODE clear_rc = ClearTrxResource(trx); + RETURN_CODE clear_rc = ClearTrxResource(trx, gen, true); if (BAIDU_UNLIKELY(clear_rc != UBRING_OK && clear_rc != UBRING_REENTRY)) { LOG(ERROR) << "Trx close callback failed, " << trx->local_shm.name << " clear trx resource failed."; break; @@ -378,7 +441,8 @@ RETURN_CODE UBRing::UbrAddHBTimer() { const uint64_t interval_us = (uint64_t)FLAGS_ub_hb_timer_interval_s * SEC_TO_USEC; RETURN_CODE rc = UbrTimerStart(&_trx->hb_timer, 0, interval_us, - UbrTrxHBCallback, (void*)_trx); + UbrTrxHBCallback, (void*)_trx, + ATOMIC_LOAD(_trx->ubr_id)); if (BAIDU_UNLIKELY(rc != UBRING_OK)) { LOG(ERROR) << "Start ubr heartbeat timer failed."; return UBRING_ERR; @@ -386,26 +450,27 @@ RETURN_CODE UBRing::UbrAddHBTimer() { return UBRING_OK; } -RETURN_CODE UBRing::UbrPassiveClearTrx(UbrTrx *trx) { +RETURN_CODE UBRing::UbrPassiveClearTrx(UbrTrx *trx, uint64_t expect_ubr_id, + bool in_timer_callback) { RETURN_CODE passive_close_check_rc = UbrTrxCloseCheck(trx); if (BAIDU_UNLIKELY(passive_close_check_rc != UBRING_OK)) { if (passive_close_check_rc == UBRING_REENTRY) { LOG(INFO) << "Passive close skipped, active close in progress, name=" << trx->local_shm.name; - return ClearTrxResource(trx); + return ClearTrxResource(trx, expect_ubr_id, in_timer_callback); } return UBRING_ERR; } trx->ubr_tx.trx_state = UBR_STATE_CLOSED; trx->ubr_rx.trx_state = UBR_STATE_CLOSED; - // Non-blocking: this may run inside the heartbeat callback itself. - UbrTimerDel(&trx->close_timer); - UbrTimerDel(&trx->hb_timer); + UbrStopTrxTimer(&trx->close_timer, in_timer_callback); + UbrStopTrxTimer(&trx->hb_timer, in_timer_callback); // Wait for in-flight IO on a one-shot timer instead of sleeping on the // timer thread. - return UbrScheduleClearTimer(trx, UbrPassiveClearCallback, UbrDoPassiveClearWork); + return UbrScheduleClearTimer(trx, expect_ubr_id, + UbrPassiveClearCallback, UbrDoPassiveClearWork); } -void* UBRing::UbrPassiveClearCallback(void* args) { +void* UBRing::UbrPassiveClearCallback(void* args, uint64_t) { auto* ctl = (UbrCleanupCtl*)args; if (BAIDU_UNLIKELY(ctl == nullptr)) { LOG(ERROR) << "Trx passive clear callback failed, ctl is null."; @@ -428,11 +493,18 @@ void* UBRing::UbrPassiveClearCallback(void* args) { return nullptr; } -void* UBRing::UbrTrxHBCallback(void* args) { +void* UBRing::UbrTrxHBCallback(void* args, uint64_t gen) { auto* trx = (UbrTrx*) args; + // UbrTrxCallbackCheck rejects a null trx (and cleared queues) before we + // dereference trx->ubr_id for the stale-generation check below. if (BAIDU_UNLIKELY(UbrTrxCallbackCheck(trx) != UBRING_OK)) { return nullptr; } + // Reject a stale fire whose trx slot was released and reused: it must not + // touch the slot's new occupant. + if (BAIDU_UNLIKELY(ATOMIC_LOAD(trx->ubr_id) != gen)) { + return nullptr; + } auto* local_data_status = (UbrDataStatusQMsg *)trx->ubr_tx.local_data_status_q.addr; auto* remote_data_status = (UbrDataStatusQMsg *)trx->ubr_rx.remote_data_status_q.addr; @@ -460,20 +532,21 @@ void* UBRing::UbrTrxHBCallback(void* args) { int fd = (int)trx->local_shm.fd; LOG(INFO) << "Ubr heartbeat, start to clear trx resource. shm_fd=" << fd << ", shm_name=" << trx->local_shm.name; - UbrPassiveClearTrx(trx); + UbrPassiveClearTrx(trx, gen, true); LOG(INFO) << "Ubr heartbeat clear trx resource finish."; return nullptr; } -RETURN_CODE UBRing::UbrAddAsynClearTimer(UbrTrx *trx) { +RETURN_CODE UBRing::UbrAddAsynClearTimer(UbrTrx *trx, uint64_t expect_ubr_id) { if (BAIDU_UNLIKELY(trx == nullptr)) { LOG(ERROR) << "Trx add close timer failed, trx is null."; return UBRING_ERR; } - return UbrScheduleClearTimer(trx, UbrAsynClearCallback, UbrDoAsynClearWork); + return UbrScheduleClearTimer(trx, expect_ubr_id, + UbrAsynClearCallback, UbrDoAsynClearWork); } -void *UBRing::UbrAsynClearCallback(void *args) +void *UBRing::UbrAsynClearCallback(void *args, uint64_t) { auto* ctl = (UbrCleanupCtl*) args; if (BAIDU_UNLIKELY(ctl == nullptr)) { @@ -1164,7 +1237,7 @@ RETURN_CODE UBRing::WritevHasEnoughSpace(size_t buf_len) return UBRING_OK; } -RETURN_CODE UBRing::UbrClearResourceCheck(UbrTrx *trx) +RETURN_CODE UBRing::UbrClearResourceCheck(UbrTrx *trx, bool in_timer_callback) { if (BAIDU_UNLIKELY(trx == nullptr)) { LOG(ERROR) << "Trx close failed, trx is null."; @@ -1180,9 +1253,10 @@ RETURN_CODE UBRing::UbrClearResourceCheck(UbrTrx *trx) local_tx_event_q->flag = UBR_STATE_CLOSING; } - // Non-blocking: may run inside the close callback itself. - UbrTimerDel(&trx->close_timer); - UbrTimerDel(&trx->hb_timer); + // Wait out a dispatched callback before the slot's trx is cleared; inside + // the callback itself only the non-blocking form is possible. + UbrStopTrxTimer(&trx->close_timer, in_timer_callback); + UbrStopTrxTimer(&trx->hb_timer, in_timer_callback); if (local_tx_event_q->flag == UBR_STATE_CLOSING) { local_tx_event_q->flag = UBR_STATE_CLOSED; @@ -1192,14 +1266,15 @@ RETURN_CODE UBRing::UbrClearResourceCheck(UbrTrx *trx) return UBRING_OK; } -RETURN_CODE UBRing::ClearTrxResource(UbrTrx *trx) +RETURN_CODE UBRing::ClearTrxResource(UbrTrx *trx, uint64_t expect_ubr_id, + bool in_timer_callback) { - RETURN_CODE rc = UbrClearResourceCheck(trx); + RETURN_CODE rc = UbrClearResourceCheck(trx, in_timer_callback); if (rc != UBRING_OK) { return rc; } - rc = UbrAddAsynClearTimer(trx); + rc = UbrAddAsynClearTimer(trx, expect_ubr_id); if (rc != UBRING_OK) { LOG(ERROR) << "Trx close, add " << trx->local_shm.name << " close clear timer failed."; return UBRING_ERR; diff --git a/src/brpc/ubshm/ub_ring.h b/src/brpc/ubshm/ub_ring.h index 16dd59e21e..1f720c54ad 100644 --- a/src/brpc/ubshm/ub_ring.h +++ b/src/brpc/ubshm/ub_ring.h @@ -49,19 +49,25 @@ class UBRing : public butil::IReader { RETURN_CODE UbrAddTimer(); - static void *UbrTrxCloseCallback(void *args); + static void *UbrTrxCloseCallback(void *args, uint64_t gen); RETURN_CODE UbrAddHBTimer(); - static void *UbrTrxHBCallback(void *args); + static void *UbrTrxHBCallback(void *args, uint64_t gen); - static RETURN_CODE UbrPassiveClearTrx(UbrTrx *trx); + // `in_timer_callback' is true only when called from UbrTrxCloseCallback or + // UbrTrxHBCallback, i.e. from a per-trx timer callback: the teardown then + // cannot wait for a dispatched sibling callback (that would join the + // running task), and does not need to, because bthread runs all timer + // callbacks on one global timer thread. + static RETURN_CODE UbrPassiveClearTrx(UbrTrx *trx, uint64_t expect_ubr_id, + bool in_timer_callback); - static RETURN_CODE UbrAddAsynClearTimer(UbrTrx *trx); + static RETURN_CODE UbrAddAsynClearTimer(UbrTrx *trx, uint64_t expect_ubr_id); - static void *UbrAsynClearCallback(void *args); + static void *UbrAsynClearCallback(void *args, uint64_t gen); - static void *UbrPassiveClearCallback(void *args); + static void *UbrPassiveClearCallback(void *args, uint64_t gen); int UbrTrxSend(const void *buf, uint32_t buf_len); @@ -214,8 +220,9 @@ class UBRing : public butil::IReader { void PreWriteAddr(uint8_t *addr, size_t len); RETURN_CODE WritevHasEnoughSpace(size_t buf_len); RETURN_CODE UbrServerTrxInit(SHM *local_shm, SHM *remote_shm); - static RETURN_CODE UbrClearResourceCheck(UbrTrx *trx); - static RETURN_CODE ClearTrxResource(UbrTrx *trx); + static RETURN_CODE UbrClearResourceCheck(UbrTrx *trx, bool in_timer_callback); + static RETURN_CODE ClearTrxResource(UbrTrx *trx, uint64_t expect_ubr_id, + bool in_timer_callback); UbrTrx* _trx{nullptr}; }; diff --git a/src/brpc/ubshm/ub_ring_manager.cpp b/src/brpc/ubshm/ub_ring_manager.cpp index 9084596bba..44e7024409 100644 --- a/src/brpc/ubshm/ub_ring_manager.cpp +++ b/src/brpc/ubshm/ub_ring_manager.cpp @@ -16,6 +16,7 @@ // under the License. #include +#include #include #include "brpc/ubshm/ub_ring.h" #include "brpc/ubshm/ub_ring_manager.h" @@ -99,6 +100,37 @@ RETURN_CODE UBRingManager::UbrMgrInit() { } void UBRingManager::UbrMgrFini() { + // The pool arrays are only safe to walk once UbrMgrInit has allocated and + // zeroed all four of them; its allocation-failure path calls this function + // with some of them still null and the others uninitialized. + const bool pool_ready = g_ubr_mgr.trx_mgr != nullptr && + g_ubr_mgr.trx_mgr_unit_status != nullptr && + g_ubr_mgr.trx_mgr_unit_id != nullptr && + g_ubr_mgr.trx_mgr_unit_ctl != nullptr; + + // Wait out close/heartbeat callbacks that are already dispatched, then + // stop the timers, before the pool is freed: UbrTimerDel alone is + // non-blocking, so a callback that already passed its generation check + // could still be reading UbrTrx when FREE_PTR(trx_mgr) runs. + // UbrTimerDelAndWait blocks until a dispatched callback has returned, but + // it must run without g_ubr_trx_mgr_mtx because the callbacks take that + // lock themselves; snapshot the slots under the lock first. + if (pool_ready) { + std::vector used_slots; + { + BAIDU_SCOPED_LOCK(g_ubr_trx_mgr_mtx); + for (uint32_t i = 0; i < g_ubr_mgr.trx_cap; ++i) { + if (g_ubr_mgr.trx_mgr_unit_status[i] == UBR_MGR_UNIT_USED) { + used_slots.push_back(i); + } + } + } + for (uint32_t i : used_slots) { + UbrTimerDelAndWait(&g_ubr_mgr.trx_mgr[i].close_timer); + UbrTimerDelAndWait(&g_ubr_mgr.trx_mgr[i].hb_timer); + } + } + // Cancel the pending delayed cleanups and wait for the in-flight ones // (each holds one extra reference) to finish, before the pool memory // they touch is freed. A ctl whose timer is still starting can only be @@ -108,7 +140,7 @@ void UBRingManager::UbrMgrFini() { busy = false; { BAIDU_SCOPED_LOCK(g_ubr_trx_mgr_mtx); - if (g_ubr_mgr.trx_mgr_unit_ctl != nullptr) { + if (pool_ready) { for (uint32_t i = 0; i < g_ubr_mgr.trx_cap; ++i) { UbrCleanupCtl* ctl = g_ubr_mgr.trx_mgr_unit_ctl[i]; if (ctl == nullptr) { @@ -130,7 +162,7 @@ void UBRingManager::UbrMgrFini() { } { BAIDU_SCOPED_LOCK(g_ubr_trx_mgr_mtx); - if (g_ubr_mgr.trx_mgr_unit_ctl != nullptr) { + if (pool_ready) { for (uint32_t i = 0; i < g_ubr_mgr.trx_cap; ++i) { UbrCleanupCtl* ctl = g_ubr_mgr.trx_mgr_unit_ctl[i]; if (ctl != nullptr) { @@ -235,6 +267,18 @@ RETURN_CODE UBRingManager::ReleaseUbrTrxFromMgr(UbrTrx *trx, return UBRING_ERR; } + // Disarm the per-acquisition periodic timers before the slot becomes + // reusable: a surviving task would keep re-arming and firing against the + // slot's next occupant. Non-blocking delete only -- this path holds + // g_ubr_trx_mgr_mtx and the timer callbacks take the same lock, so + // UbrTimerDelAndWait would deadlock; it is not needed either, because + // every caller has already quiesced the per-trx callbacks before reaching + // here: the clear paths wait through UbrStopTrxTimer (or run on the single + // bthread timer thread) before they call ReleaseUbrTrxFromMgr, the force + // close waits in UbrTrxClose, and UbrMgrFini waits for all of them. + UbrTimerDel(&trx->close_timer); + UbrTimerDel(&trx->hb_timer); + // Mutate the trx only after the generation check passed. trx->local_shm.addr = nullptr; trx->ubr_tx.local_tx_event_q.addr = nullptr; @@ -391,7 +435,8 @@ int32_t UBRingManager::UbEventCallback(const char *shm_name) ++g_ub_event_cnt; int fd = (int)g_ubr_mgr.trx_mgr[i].local_shm.fd; LOG(WARNING) << "Ub event callback, the fd of the faulty link is " << fd; - return UBRing::UbrPassiveClearTrx(&g_ubr_mgr.trx_mgr[i]); + const uint64_t expect_ubr_id = ATOMIC_LOAD(g_ubr_mgr.trx_mgr[i].ubr_id); + return UBRing::UbrPassiveClearTrx(&g_ubr_mgr.trx_mgr[i], expect_ubr_id, false); } } return UBRING_ERR; From 5f55f69a0fc1c478412a10495643f16c3badb17a Mon Sep 17 00:00:00 2001 From: Sherlock0203 <1297399478@qq.com> Date: Wed, 30 Sep 2026 16:10:42 +0800 Subject: [PATCH 5/9] fix(ubshm): wait for self-deleted timers and serialize pool teardown Address the second ubring review round on the bthread-timer refactor. Timer facade: - A periodic callback that deletes its own timer now only marks the task stopped and leaves it anchored in the handle slot until the callback returns; UbrTimerOnFire retires the slot on exit. Previously the callback pulled the task out of the slot first, so a concurrent UbrTimerDelAndWait returned immediately and the caller could clear a trx whose callback was still running. - UbrTimerDelAndWait recognizes that the caller is the callback of that very timer and degrades to the non-blocking delete, which removes the in_timer_callback argument that every teardown entry point had to thread correctly (and could get wrong). - UbrTimerStart (one-shot) and UbrTimerStartPeriodic are now separate functions, and a periodic start rejects interval_us == 0 instead of silently arming a one-shot: that dropped the close check, the heartbeat or the shm drain for the rest of the connection's life. The close and hb call sites use the periodic form and validate their flags; the shm drain period is derived to stay positive because its flag legitimately accepts 0 for the one-shot delayed-clear. - A back-off function returning 0 keeps the previous interval: re-arming for "now" would let one timer monopolize the process-wide timer thread. - The two join waits poll every 100us instead of 1ms; they only run on teardown paths. Cleanup-control publication and trx close: - UBRingManager::TryClaimTrxClose validates the slot generation and moves close_cnt in one critical section, so a close started from a stale snapshot cannot mark the slot's new occupant CLOSED or delete its timers. UbEventCallback looks the faulty link up before it closes it, so this was reachable through the ubsmem fault callback; UbrTrxCloseCheck takes expect_ubr_id and is now only a wrapper around the claim. - TryPublishUnitCleanupCtl publishes the cleanup control object on the trx and anchors it in the pool slot under one lock, and DetachUnitCleanupCtl clears both, so a force close never observes a trx-side publication without its anchor. The inline fallback gates the cleanup on IsUbrTrxSlotUsed instead of ubr_id: ReleaseUbrTrxFromMgr does not change ubr_id, so the old check let a released slot be cleaned twice. Shutdown: - UbrMgrFini takes a shutdown barrier before anything else: acquisitions, timer arming (ArmTimersExclusive, now used by UbrAddTimer) and cleanup publications are refused, and the faulty-shm callback registers through BeginPoolAccess, which Fini waits for before the pool is freed. Because arming is serialized by the same lock, the timers armed before the flag are the complete set and one wait pass replaces the retry loop whose window was never actually closed. - The shm list is torn down through a handle: DestroyShmTimer takes ShmList** and clears it, so g_shm_list is never left dangling, a repeated teardown is a no-op and UbsShmFini is idempotent, which makes a repeated ShmMgrFini safe instead of a double free. UbrMgrInit value-initializes the pool instead of memset'ing it, matching the placement-new UbrTrx construction the style commit introduced (UbrTrx holds butil::atomic members). Tested: cmake --build cmake-build-debug --target brpc_ubring_unittest, and ./test/brpc_ubring_unittest (12/12 passed). --- docs/cn/ubring.md | 2 +- docs/en/ubring.md | 2 +- src/brpc/ubshm/shm/shm_ubs.cpp | 61 +++++++-- src/brpc/ubshm/shm/shm_ubs.h | 5 +- src/brpc/ubshm/timer/timer_mgr.cpp | 102 ++++++++++++-- src/brpc/ubshm/timer/timer_mgr.h | 64 ++++++--- src/brpc/ubshm/ub_ring.cpp | 212 +++++++++++++++++------------ src/brpc/ubshm/ub_ring.h | 31 +++-- src/brpc/ubshm/ub_ring_manager.cpp | 179 +++++++++++++++++++++--- src/brpc/ubshm/ub_ring_manager.h | 75 ++++++++-- 10 files changed, 560 insertions(+), 173 deletions(-) diff --git a/docs/cn/ubring.md b/docs/cn/ubring.md index cd100db84d..f4c7c0b841 100644 --- a/docs/cn/ubring.md +++ b/docs/cn/ubring.md @@ -187,7 +187,7 @@ UBRing 架构包含以下组件： ### 定时器管理 -UBRing 的连接管理和超时处理基于 bthread 定时器（`bthread_timer_add`/`bthread_timer_del`，由 `timer_mgr.cpp` 封装）。非阻塞的 `UbrTimerDel` 可在回调内自删；销毁回调参数所属资源的外部路径用 `UbrTimerDelAndWait` 等待运行中回调退出；一次性定时器触发后自动清理句柄。定时器回调运行在进程全局的 bthread 定时线程上，必须快速返回。关闭检查定时器在链路空闲时按指数退避轮询（上限 `ub_event_queue_timer_interval_max_us`，默认 10ms），有流量或正在关闭时恢复快速轮询。 +UBRing 的连接管理和超时处理基于 bthread 定时器（`bthread_timer_add`/`bthread_timer_del`，由 `timer_mgr.cpp` 封装）。非阻塞的 `UbrTimerDel` 可在回调内自删，此时句柄会保留到该回调返回，因此并发的外部 `UbrTimerDelAndWait` 一定能等到它；销毁回调参数所属资源的外部路径用 `UbrTimerDelAndWait` 等待运行中回调退出；一次性定时器触发后自动清理句柄。进入关闭处理时会带上调用方看到的代际，并在 manager 锁内原子认领槽位，因此按旧快照发起的关闭（例如共享内存故障事件回调）不会误伤释放后被复用的新连接。定时器回调运行在进程全局的 bthread 定时线程上，必须快速返回。关闭检查定时器在链路空闲时按指数退避轮询（上限 `ub_event_queue_timer_interval_max_us`，默认 10ms），有流量或正在关闭时恢复快速轮询。 ## 参考资料 diff --git a/docs/en/ubring.md b/docs/en/ubring.md index 0cb3d08954..9247360e04 100644 --- a/docs/en/ubring.md +++ b/docs/en/ubring.md @@ -186,7 +186,7 @@ The shared memory manager (`shm_mgr.cpp`) provides a unified interface for diffe ### Timer Management -UBRing connection management and timeout handling are built on bthread timers (`bthread_timer_add`/`bthread_timer_del`, wrapped in `timer_mgr.cpp`). The non-blocking `UbrTimerDel` is safe to call from within a timer callback; external teardown that frees resources reachable from the callback argument uses `UbrTimerDelAndWait` to wait out a possibly running callback; one-shot timers clean up their handle automatically when they fire. Timer callbacks run on the process-wide bthread timer thread and must return quickly. The close-check timer backs off exponentially while the link is idle (capped by `ub_event_queue_timer_interval_max_us`, 10ms by default) and returns to fast polling once there is traffic or a close in progress. +UBRing connection management and timeout handling are built on bthread timers (`bthread_timer_add`/`bthread_timer_del`, wrapped in `timer_mgr.cpp`). The non-blocking `UbrTimerDel` is safe to call from within a timer callback; a periodic callback that deletes its own timer keeps its handle anchored until the callback returns, so a concurrent `UbrTimerDelAndWait` from a teardown path always waits for it. External teardown that frees resources reachable from the callback argument uses `UbrTimerDelAndWait` to wait out a possibly running callback; one-shot timers clean up their handle automatically when they fire. Entering a close carries the generation the caller saw and claims the pool slot atomically under the manager lock, so a close started from a stale snapshot (the shared-memory fault event, for instance) cannot hit a connection that reused the slot in the meantime. Timer callbacks run on the process-wide bthread timer thread and must return quickly. The close-check timer backs off exponentially while the link is idle (capped by `ub_event_queue_timer_interval_max_us`, 10ms by default) and returns to fast polling once there is traffic or a close in progress. ## References diff --git a/src/brpc/ubshm/shm/shm_ubs.cpp b/src/brpc/ubshm/shm/shm_ubs.cpp index 193ba52a96..aa4086619d 100644 --- a/src/brpc/ubshm/shm/shm_ubs.cpp +++ b/src/brpc/ubshm/shm/shm_ubs.cpp @@ -49,6 +49,10 @@ DEFINE_int32(ub_flying_io_timeout_s, 5, char g_region_name[MAX_REGION_NAME_DESC_LENGTH] = {0}; butil::atomic g_shm_timer_id(nullptr); ShmList *g_shm_list = nullptr; +// Set by UbsShmFini, cleared by UbsShmInit: ShmMgrFini may run more than once, +// and the SDK finalize plus the shm list teardown must happen exactly once per +// init. +static bool g_ubs_shm_finalized = false; static RETURN_CODE UbsShmInterfacesLoad(void); char hostname[MAX_HOST_NAME_DESC_LENGTH]; @@ -314,6 +318,9 @@ void UbsMemLoggerPrint(int level, const char *msg) RETURN_CODE UbsShmInit(void) { + // A new init makes UbsShmFini run its teardown again. + g_ubs_shm_finalized = false; + // load libubsm_sdk.so and get function pointer RETURN_CODE ret_code = UbsShmInterfacesLoad(); if (ret_code != UBRING_OK) { @@ -377,8 +384,16 @@ RETURN_CODE UbsShmInit(void) RETURN_CODE UbsShmFini(void) { + // Idempotent: ShmMgrFini is allowed to run more than once (GlobalRelease + // plus an explicit caller), and finalizing the SDK or tearing the shm list + // down twice would touch freed memory. + if (g_ubs_shm_finalized) { + return UBRING_OK; + } + g_ubs_shm_finalized = true; + // Stop the cleanup timer before finalizing the SDK it calls into. - if (BAIDU_UNLIKELY(DestroyShmTimer(g_shm_list) != UBRING_OK)) { + if (BAIDU_UNLIKELY(DestroyShmTimer(&g_shm_list) != UBRING_OK)) { LOG(ERROR) << "Ubs shm list finalize failed."; return UBRING_ERR; } @@ -459,9 +474,26 @@ void *UbsShmCallback(void* args, uint64_t) RETURN_CODE UbsShmAddTimer(ShmList *shm_list) { - const uint64_t timer_interval_us = (uint64_t)FLAGS_ub_flying_io_timeout_s * SEC_TO_USEC; - RETURN_CODE rc = UbrTimerStart(&g_shm_timer_id, 0, timer_interval_us, - UbsShmCallback, (void*)shm_list, 0); + // This timer is periodic -- it drains one pending unmap per fire -- while + // the same flag is also the one-shot delay of the delayed-clear path, where + // 0 legitimately means "do not wait for in-flight IO". A periodic timer + // needs a positive period, so fall back to a drain period of one second + // instead of either refusing to start (which would break UBS shm init for a + // valid tuning of the other use) or degrading to a single drain. + uint64_t period_s = (uint64_t)FLAGS_ub_flying_io_timeout_s; + if (BAIDU_UNLIKELY(FLAGS_ub_flying_io_timeout_s <= 0)) { + LOG(WARNING) << "ub_flying_io_timeout_s=" << FLAGS_ub_flying_io_timeout_s + << " disables the delayed-clear wait; using a 1s drain period " + << "for the shm cleanup timer."; + period_s = 1; + } + const uint64_t timer_interval_us = period_s * SEC_TO_USEC; + // The generation argument is the caller's staleness guard; UbsShmCallback + // ignores it and never frees `shm_list', whose lifetime is settled by + // DestroyShmTimer waiting through UbrTimerDelAndWait before it tears the + // list down. 0 is therefore deliberate here, not a missing generation. + RETURN_CODE rc = UbrTimerStartPeriodic(&g_shm_timer_id, 0, timer_interval_us, + UbsShmCallback, (void*)shm_list, 0); if (BAIDU_UNLIKELY(rc != UBRING_OK)) { LOG(ERROR) << "Start shm timer failed."; return UBRING_ERR; @@ -495,15 +527,23 @@ RETURN_CODE InitShmTimer(ShmList **shm_list) return UBRING_OK; } -RETURN_CODE DestroyShmTimer(ShmList *shm_list) +RETURN_CODE DestroyShmTimer(ShmList **shm_list) { // Wait out a possibly running UbsShmCallback before tearing shm_list down. UbrTimerDelAndWait(&g_shm_timer_id); - if (shm_list == nullptr) { - LOG(WARNING) << "Shm list is null."; + if (BAIDU_UNLIKELY(shm_list == nullptr)) { + LOG(ERROR) << "Shm list handle is null."; return UBRING_ERR; } - ShmListNode* current = shm_list->head; + if (*shm_list == nullptr) { + return UBRING_OK; // already destroyed, idempotent + } + + // Own the pointer for the whole teardown, and clear the caller's handle + // (FREE_PTR does that for us) before returning: leaving it dangling lets a + // second ShmMgrFini/AddShmToList use freed memory. + ShmList* list = *shm_list; + ShmListNode* current = list->head; ShmListNode* next; while (current != nullptr) { @@ -511,8 +551,9 @@ RETURN_CODE DestroyShmTimer(ShmList *shm_list) free(current); current = next; } - pthread_mutex_destroy(&shm_list->shm_lock); - FREE_PTR(shm_list); + pthread_mutex_destroy(&list->shm_lock); + FREE_PTR(*shm_list); + LOG(INFO) << "Ubs shm list destroyed."; return UBRING_OK; } diff --git a/src/brpc/ubshm/shm/shm_ubs.h b/src/brpc/ubshm/shm/shm_ubs.h index 39f622cf76..02ed51cd46 100644 --- a/src/brpc/ubshm/shm/shm_ubs.h +++ b/src/brpc/ubshm/shm/shm_ubs.h @@ -44,7 +44,10 @@ void UbsMemLoggerPrint(int level, const char *msg); void *UbsShmCallback(void* args, uint64_t gen); RETURN_CODE UbsShmAddTimer(ShmList *shm_list); RETURN_CODE InitShmTimer(ShmList **shm_list); -RETURN_CODE DestroyShmTimer(ShmList *shm_list); +// Takes the handle so it can be cleared after the list is freed: a caller that +// keeps the pointer (g_shm_list) must not be left with a dangling one. Calling +// it again on a cleared handle is a no-op. +RETURN_CODE DestroyShmTimer(ShmList **shm_list); RETURN_CODE AddShmToList(ShmList *shm_list, SHM *shm); RETURN_CODE IsExistInShmList(ShmList *shm_list, const SHM *shm); } diff --git a/src/brpc/ubshm/timer/timer_mgr.cpp b/src/brpc/ubshm/timer/timer_mgr.cpp index 28a5d74f7c..344c365a9f 100644 --- a/src/brpc/ubshm/timer/timer_mgr.cpp +++ b/src/brpc/ubshm/timer/timer_mgr.cpp @@ -33,6 +33,11 @@ enum UbrTimerState { kDead = 2 // scheduling failed }; +// Granularity of the two join waits in UbrTimerDelAndWait. They only run on +// teardown paths and only for the short window in which a timer is being +// started or a callback is finishing. +constexpr uint64_t kTimerPollIntervalUs = 100; + } // namespace // Reference rules: one "owner" ref for the handle slot, one "schedule" ref @@ -45,6 +50,13 @@ enum UbrTimerState { // the callback so that the callback may free the object storing the slot. // All atomics are seq_cst so no interleaving can release a ref twice or // free the task while a callback or the starter still touches it. +// +// A periodic callback that deletes its own timer does NOT take the task out +// of the slot: it only marks the task stopped, and the firing callback +// retires the slot right before it returns (see UbrTimerOnFire). Keeping the +// task anchored for the whole callback is what lets a concurrent +// UbrTimerDelAndWait still find it and wait for the callback to finish, +// instead of returning as if no callback were in flight. struct UbrTimerTask { butil::atomic* slot; butil::atomic id; @@ -63,6 +75,11 @@ struct UbrTimerTask { namespace { +// The task whose callback is running on this thread, if any. It lets +// UbrTimerDel recognize a delete issued by the callback's own dispatch and +// leave the task anchored until the callback returns. +thread_local UbrTimerTask* t_running_task = nullptr; + void ReleaseRef(UbrTimerTask* task) { if (task->ref.fetch_sub(1) == 1) { if (task->join_pending.load()) { @@ -78,17 +95,41 @@ void UbrTimerOnFire(void* p) { if (task->periodic) { if (!task->stopped.load()) { + // Publish the running task so an UbrTimerDel issued by this very + // callback leaves the slot anchored for a concurrent waiter. + UbrTimerTask* prev_running = t_running_task; + t_running_task = task; task->cb(task->arg, task->gen); + t_running_task = prev_running; } // Claim the next schedule's ref before re-reading `stopped' so a // racing delete can neither free the task nor orphan a re-arm. task->ref.fetch_add(1); if (task->stopped.load()) { - ReleaseRef(task); + // Retire the slot here, on the callback's own exit: a self-delete + // deliberately left the task anchored. A deleter that took the + // slot instead has already consumed the owner reference. + UbrTimerId anchored = task; + if (task->slot->compare_exchange_strong(anchored, nullptr)) { + ReleaseRef(task); // owner + } + ReleaseRef(task); // claimed next schedule } else { uint64_t interval = task->interval_us; if (task->backoff != nullptr) { - interval = task->backoff(task->arg, interval); + const uint64_t next = task->backoff(task->arg, interval); + if (BAIDU_UNLIKELY(next == 0)) { + // Re-arming for "now" would let this one timer monopolize + // the process-wide timer thread. A back-off (or a flag it + // reads) returning 0 is a bug, so keep the previous + // interval instead of honouring it. Guarded here rather + // than in each back-off function, because any of them can + // make this mistake. + LOG_EVERY_SECOND(ERROR) << "Ubr timer back-off returned 0, keeping interval_us=" + << interval; + } else { + interval = next; + } task->interval_us = interval; } bthread_timer_t id = 0; @@ -133,13 +174,21 @@ UbrTimerTask* TakeOutTask(butil::atomic* slot) { } RETURN_CODE TimerStartInternal(butil::atomic* slot, uint64_t delay_us, - uint64_t interval_us, void* (*cb)(void*, uint64_t), + uint64_t interval_us, bool periodic, + void* (*cb)(void*, uint64_t), void* arg, uint64_t gen, UbrTimerBackoffFn backoff) { if (BAIDU_UNLIKELY(slot == nullptr || cb == nullptr)) { LOG(ERROR) << "Ubr timer start invalid argument, slot=" << slot; return UBRING_ERR; } + if (BAIDU_UNLIKELY(periodic && interval_us == 0)) { + // Rejecting beats silently arming a one-shot: a caller that asks for a + // periodic timer and gets a single fire loses whatever the timer was + // watching for the rest of the connection's life. + LOG(ERROR) << "Ubr periodic timer start requires a positive interval."; + return UBRING_ERR; + } UbrTimerTask* task = new (std::nothrow) UbrTimerTask(); if (BAIDU_UNLIKELY(task == nullptr)) { @@ -153,7 +202,7 @@ RETURN_CODE TimerStartInternal(butil::atomic* slot, uint64_t delay_u task->gen = gen; task->backoff = backoff; task->interval_us = interval_us; - task->periodic = (interval_us > 0); + task->periodic = periodic; task->state.store(kStarting); task->stopped.store(false); task->ref.store(3); // owner + schedule + starter @@ -199,15 +248,35 @@ RETURN_CODE TimerStartInternal(butil::atomic* slot, uint64_t delay_u } // namespace RETURN_CODE UbrTimerStart(butil::atomic* slot, uint64_t delay_us, - uint64_t interval_us, void* (*cb)(void*, uint64_t), - void* arg, uint64_t gen, UbrTimerBackoffFn backoff) { - return TimerStartInternal(slot, delay_us, interval_us, cb, arg, gen, backoff); + void* (*cb)(void*, uint64_t), void* arg, uint64_t gen) { + return TimerStartInternal(slot, delay_us, 0, false, cb, arg, gen, nullptr); +} + +RETURN_CODE UbrTimerStartPeriodic(butil::atomic* slot, + uint64_t delay_us, uint64_t interval_us, + void* (*cb)(void*, uint64_t), void* arg, + uint64_t gen, UbrTimerBackoffFn backoff) { + return TimerStartInternal(slot, delay_us, interval_us, true, cb, arg, gen, + backoff); } int UbrTimerDel(butil::atomic* slot) { if (slot == nullptr) { return 1; } + // A periodic callback deleting its own timer must not pull the task out of + // the slot: the anchored task is what lets a concurrent UbrTimerDelAndWait + // find the running callback and wait for it. Mark it stopped instead and + // let UbrTimerOnFire retire the slot when the callback returns. Deleting a + // sibling timer from inside a callback is unaffected: the slots differ, so + // the pointer comparison below fails and the normal path runs. + if (t_running_task != nullptr && slot->load() == t_running_task) { + t_running_task->stopped.store(true); + // 0 means "the timer is stopped", not "the handle slot is now free": + // the slot is retired by UbrTimerOnFire when this callback returns, and + // a concurrent UbrTimerDelAndWait may still be joining the task. + return 0; + } // Take the ownership of the slot first: after this exchange every // dereference below is safe (the task cannot be freed while we hold // the owner reference the slot used to anchor). @@ -243,6 +312,15 @@ void UbrTimerDelAndWait(butil::atomic* slot) { if (slot == nullptr) { return; } + // Called by the callback of that very timer: joining it would wait for + // ourselves, so degrade to the non-blocking delete (UbrTimerDel marks the + // task stopped and lets UbrTimerOnFire retire the slot on callback exit). + // Callers therefore do not need to know whether they run on the timer + // thread. + if (t_running_task != nullptr && slot->load() == t_running_task) { + UbrTimerDel(slot); + return; + } UbrTimerTask* task = TakeOutTask(slot); if (task == nullptr) { return; @@ -250,9 +328,13 @@ void UbrTimerDelAndWait(butil::atomic* slot) { task->join_pending.store(true); task->stopped.store(true); // A start still in flight cannot be cancelled yet; wait for the - // starter to schedule it or mark it dead. + // starter to schedule it or mark it dead, polling at + // kTimerPollIntervalUs. DelAndWait runs on teardown paths (force close, + // UbrMgrFini, connect failure) where a sub-millisecond delay is + // irrelevant, so polling keeps the facade free of a second wait primitive + // and works identically from bthreads and plain pthreads. while (task->state.load() == kStarting) { - bthread_usleep(1000); + bthread_usleep(kTimerPollIntervalUs); } if (task->state.load() == kScheduled) { bthread_timer_t id = task->id.load(); @@ -262,7 +344,7 @@ void UbrTimerDelAndWait(butil::atomic* slot) { } ReleaseRef(task); // owner reference while (!task->done.load()) { - bthread_usleep(1000); + bthread_usleep(kTimerPollIntervalUs); } task->join_pending.store(false); delete task; diff --git a/src/brpc/ubshm/timer/timer_mgr.h b/src/brpc/ubshm/timer/timer_mgr.h index 04b43c02b0..b8d3676ab6 100644 --- a/src/brpc/ubshm/timer/timer_mgr.h +++ b/src/brpc/ubshm/timer/timer_mgr.h @@ -35,20 +35,33 @@ typedef struct UbrTimerTask* UbrTimerId; // Runs on the timer thread only. typedef uint64_t (*UbrTimerBackoffFn)(void* arg, uint64_t cur_interval_us); -// Schedule `cb(arg, gen)' to run after `delay_us' and, when `interval_us' > 0, -// re-arm itself after every run until deleted. One-shot timers release -// their handle slot before running the callback, so the callback may free -// the object that stores the slot; the task object itself is released -// automatically. `slot' must point to a real butil::atomic -// object, so start and delete can race on one RMW without reinterpreting -// plain storage as an atomic. `gen' is an opaque value carried by the task -// and passed back on every fire, so a callback armed for an object that was -// later released and reused can detect that it is stale. It is required: -// passing a wrong value silently disables the generation guard. +// Schedule a one-shot `cb(arg, gen)' to run after `delay_us'. The handle slot +// is released before the callback runs, so the callback may free the object +// that stores the slot; the task object itself is released automatically. +// `slot' must point to a real butil::atomic object, so start and +// delete can race on one RMW without reinterpreting plain storage as an +// atomic. `gen' is an opaque value carried by the task and passed back on the +// fire, so a callback armed for an object that was later released and reused +// can detect that it is stale. It is required: passing a wrong value silently +// disables the generation guard, so callers must pass the generation of the +// object `arg' points to. RETURN_CODE UbrTimerStart(butil::atomic* slot, uint64_t delay_us, - uint64_t interval_us, void* (*cb)(void*, uint64_t), - void* arg, uint64_t gen, - UbrTimerBackoffFn backoff = nullptr); + void* (*cb)(void*, uint64_t), void* arg, uint64_t gen); + +// Schedule `cb(arg, gen)' after `delay_us' and re-arm it every `interval_us' +// until it is deleted. `interval_us' must be positive: a periodic timer without +// a period is a misconfiguration and is rejected here, instead of silently +// degrading to a one-shot (which is what the interval-based API used to do). +// Unlike a one-shot timer, a periodic one retires its handle slot only after +// the callback returned, so the callback must NOT free the object that stores +// the slot. `backoff' maps the interval the timer is currently using to the +// next one; returning 0 is ignored (the previous interval is kept) so a broken +// back-off cannot turn the global timer thread into a spin loop. +RETURN_CODE UbrTimerStartPeriodic(butil::atomic* slot, + uint64_t delay_us, uint64_t interval_us, + void* (*cb)(void*, uint64_t), void* arg, + uint64_t gen, + UbrTimerBackoffFn backoff = nullptr); // Non-blocking delete, safe to call from inside the timer callback itself. // This function does not wait for an already running callback and does not @@ -60,6 +73,11 @@ RETURN_CODE UbrTimerStart(butil::atomic* slot, uint64_t delay_us, // dispatched or running callback may still execute once more. Callers // must not reclaim resources reachable from `arg` based on this return // alone; use UbrTimerDelAndWait when teardown needs to wait for callbacks. +// A return of 0 therefore never means "the handle is free again". +// A periodic callback that deletes its own timer falls into the periodic case +// as well, but only marks it stopped: the handle slot stays anchored until the +// callback returns, so a concurrent UbrTimerDelAndWait on the same slot really +// does wait for that callback. // // Returns 1 when this caller did not acquire timer ownership. The callback, // another deleter, or the scheduling path is responsible for settling the @@ -67,12 +85,20 @@ RETURN_CODE UbrTimerStart(butil::atomic* slot, uint64_t delay_us, int UbrTimerDel(butil::atomic* slot); // Delete and wait until a possibly running callback finished, so the -// caller can free resources reachable from `arg'. Never call this on the -// callback's own task. A one-shot callback that is already running holds -// the ownership of `arg' by itself (mirroring bthread_timer_del returning -// 1) and cannot be waited for through the slot. The wait polls with -// bthread_usleep, which degrades to ::usleep on plain pthread callers -// (e.g. process-exit paths). +// caller can free resources reachable from `arg'. Safe to call from inside the +// callback's own dispatch: it then degrades to the non-blocking UbrTimerDel +// instead of joining itself, so callers never have to know which thread they +// run on. +// +// A one-shot callback that is already running holds the ownership of `arg' by +// itself (mirroring bthread_timer_del returning 1) and cannot be waited for +// through the slot. Callers of one-shot timers must therefore keep `arg' alive +// by their own reference (this is a hard requirement, not a convenience: the +// delayed-clear path relies on UbrCleanupCtl's reference count for exactly +// this reason). A periodic callback -- including one that deleted its own +// timer -- always can be waited for, because its task stays anchored until it +// returns. The wait polls with bthread_usleep, which degrades to ::usleep on +// plain pthread callers (e.g. process-exit paths). void UbrTimerDelAndWait(butil::atomic* slot); } // namespace ubring diff --git a/src/brpc/ubshm/ub_ring.cpp b/src/brpc/ubshm/ub_ring.cpp index 1d9b6bbc36..ae1ea56a49 100644 --- a/src/brpc/ubshm/ub_ring.cpp +++ b/src/brpc/ubshm/ub_ring.cpp @@ -68,24 +68,18 @@ RETURN_CODE UBRing::UbrTrxMapShm(SHM *local_shm, SHM *remote_shm) return UBRING_OK; } -// Stop a per-trx timer before its slot becomes reusable. Outside a per-trx -// timer callback this waits -- UbrTimerDelAndWait returns only after a -// callback that was already dispatched has left the trx -- which is what lets -// the caller clear the trx and free its shared memory afterwards. Inside the -// callback itself the wait would join the task that is currently running, so -// the non-blocking delete is used instead; no wait is needed there, because -// bthread dispatches every timer callback from one global timer thread +// Stop a per-trx timer before its slot becomes reusable. UbrTimerDelAndWait +// returns only after a callback that was already dispatched has left the trx, +// which is what lets the caller clear the trx and free its shared memory +// afterwards. When the caller *is* the callback of that very timer, the facade +// recognizes it and degrades to the non-blocking delete (waiting would join the +// calling callback), so no caller has to know whether it runs on the timer +// thread. A sibling timer callback cannot be running concurrently either, +// because bthread dispatches every timer callback from one global timer thread // (TimerThread in bthread/timer_thread.{h,cpp} is created by a single -// pthread_once), so this trx's sibling timer callback cannot be running -// concurrently and the non-blocking delete already removed it from the timer -// heap. -static void UbrStopTrxTimer(butil::atomic* slot, - bool in_timer_callback) { - if (in_timer_callback) { - UbrTimerDel(slot); - } else { - UbrTimerDelAndWait(slot); - } +// pthread_once). +static void UbrStopTrxTimer(butil::atomic* slot) { + UbrTimerDelAndWait(slot); } static void UbrDoAsynClearWork(UbrTrx *trx, uint64_t expect_ubr_id) { @@ -144,24 +138,24 @@ static RETURN_CODE UbrScheduleClearTimer(UbrTrx *trx, uint64_t expect_ubr_id, // manager anchor is taken by // TryPublishUnitCleanupCtl - UbrCleanupCtl* expected = nullptr; - if (!trx->cleanup_ctl.compare_exchange_strong(expected, ctl)) { - delete ctl; // another schedule won - return UBRING_OK; - } + // The manager publishes the ctl on the trx and anchors it in the pool slot + // in one critical section, so a concurrent force close can never observe + // the trx-side publication without its anchor. if (!UBRingManager::TryPublishUnitCleanupCtl(trx->trx_mgr_index, ctl->ubr_id, ctl)) { - // The slot was released (and possibly reused) before we could - // anchor: force close or the new occupant owns it now. Nothing - // was armed yet -- just undo the trx-side publication. - expected = ctl; - trx->cleanup_ctl.compare_exchange_strong(expected, nullptr); + // Another schedule won the slot, or the slot was released (and + // possibly reused) before we could anchor: force close or the new + // occupant owns it now. Nothing was armed or published -- drop our + // two references. ctl->ReleaseRef(); // timer/callback reference, never armed ctl->ReleaseRef(); // starter reference return UBRING_OK; } + // One-shot on purpose: this only delays the cleanup so in-flight IO can + // drain, and FLAGS_ub_flying_io_timeout_s == 0 legitimately means "do not + // delay". RETURN_CODE rc = UbrTimerStart(&ctl->timer, - (uint64_t)FLAGS_ub_flying_io_timeout_s * SEC_TO_USEC, 0, cb, ctl, + (uint64_t)FLAGS_ub_flying_io_timeout_s * SEC_TO_USEC, cb, ctl, expect_ubr_id); if (BAIDU_UNLIKELY(rc != UBRING_OK)) { // The timer was never scheduled: this path owns the manager, @@ -171,7 +165,10 @@ static RETURN_CODE UbrScheduleClearTimer(UbrTrx *trx, uint64_t expect_ubr_id, // ownership meanwhile, leave the manager anchor to it. int state_expected = UBR_CLEANUP_PENDING; if (ATOMIC_COMPARE_EXCHANGE_STRONG(ctl->state, state_expected, UBR_CLEANUP_RUNNING)) { - if (ATOMIC_LOAD(trx->ubr_id) == ctl->ubr_id) { + // Gate on the slot still being used by this generation: force + // close may have claimed the cleanup and released the slot (which + // already freed the trx resources) after we were anchored. + if (UBRingManager::IsUbrTrxSlotUsed(trx->trx_mgr_index, ctl->ubr_id)) { work(trx, ctl->ubr_id); } ATOMIC_STORE(ctl->state, UBR_CLEANUP_DONE); @@ -179,8 +176,6 @@ static RETURN_CODE UbrScheduleClearTimer(UbrTrx *trx, uint64_t expect_ubr_id, // anchor and cleanup_ctl must keep telling a concurrent force // close that this cleanup is owned (RUNNING/DONE), otherwise it // would fall into its no-ctl branch and clean the trx again. - UbrCleanupCtl* published = ctl; - trx->cleanup_ctl.compare_exchange_strong(published, nullptr); UBRingManager::DetachUnitCleanupCtl(trx->trx_mgr_index, ctl); } ctl->ReleaseRef(); // timer/callback reference @@ -192,10 +187,6 @@ static RETURN_CODE UbrScheduleClearTimer(UbrTrx *trx, uint64_t expect_ubr_id, if (UbrTimerDel(&ctl->timer) == 0) { ctl->ReleaseRef(); // timer/callback reference } - UbrCleanupCtl* published = ctl; - if (!trx->cleanup_ctl.compare_exchange_strong(published, nullptr)) { - published = nullptr; - } UBRingManager::DetachUnitCleanupCtl(trx->trx_mgr_index, ctl); ctl->ReleaseRef(); // starter reference return UBRING_OK; @@ -205,7 +196,8 @@ static RETURN_CODE UbrScheduleClearTimer(UbrTrx *trx, uint64_t expect_ubr_id, } RETURN_CODE UBRing::UbrTrxClose() { - RETURN_CODE close_check_rc = UbrTrxCloseCheck(_trx); + const uint64_t expect_ubr_id = ATOMIC_LOAD(_trx->ubr_id); + RETURN_CODE close_check_rc = UbrTrxCloseCheck(_trx, expect_ubr_id); if (BAIDU_UNLIKELY(close_check_rc != UBRING_OK)) { if (close_check_rc == UBRING_REENTRY) { LOG(INFO) << "Trx close skipped, already closing, local name=" << _trx->local_shm.name; @@ -213,7 +205,6 @@ RETURN_CODE UBRing::UbrTrxClose() { } return UBRING_ERR; } - const uint64_t expect_ubr_id = ATOMIC_LOAD(_trx->ubr_id); if (_trx->ubr_rx.remote_tx_event_q.addr != nullptr) { ((UbrEventQMsg *)_trx->ubr_rx.remote_tx_event_q.addr)->flag = UBR_STATE_CLOSING; } @@ -281,7 +272,7 @@ RETURN_CODE UBRing::UbrTrxClose() { } _trx->ubr_rx.trx_state = UBR_STATE_CLOSED; RETURN_CODE rc; - if (BAIDU_UNLIKELY((rc = ClearTrxResource(_trx, expect_ubr_id, false)) != UBRING_OK)) { + if (BAIDU_UNLIKELY((rc = ClearTrxResource(_trx, expect_ubr_id)) != UBRING_OK)) { if (rc == UBRING_REENTRY) { LOG(INFO) << "Trx close, peer is closing, trx local name=" << _trx->local_shm.name; return UBRING_OK; @@ -333,13 +324,20 @@ RETURN_CODE UBRing::UbrAddCloseTimer() { return UBRING_ERR; } + // Name the offending flag: the facade would also reject a zero interval, + // but only this log can point at the configuration that produced it. + if (BAIDU_UNLIKELY(FLAGS_ub_event_queue_timer_interval_us <= 0)) { + LOG(ERROR) << "Start ubr close timer failed, ub_event_queue_timer_interval_us=" + << FLAGS_ub_event_queue_timer_interval_us << " must be positive."; + return UBRING_ERR; + } const uint32_t interval_us = FLAGS_ub_event_queue_timer_interval_us; _trx->close_chk_in_io_id = ATOMIC_LOAD(_trx->ubr_rx.in_io_id); _trx->close_chk_out_io_id = ATOMIC_LOAD(_trx->ubr_tx.out_io_id); - RETURN_CODE rc = UbrTimerStart(&_trx->close_timer, 0, interval_us, - UbrTrxCloseCallback, (void*)_trx, - ATOMIC_LOAD(_trx->ubr_id), - UbrCloseTimerBackoff); + RETURN_CODE rc = UbrTimerStartPeriodic(&_trx->close_timer, 0, interval_us, + UbrTrxCloseCallback, (void*)_trx, + ATOMIC_LOAD(_trx->ubr_id), + UbrCloseTimerBackoff); if (BAIDU_UNLIKELY(rc != UBRING_OK)) { LOG(ERROR) << "Start ubr close timer failed, trx local name=" << _trx->local_shm.name; return UBRING_ERR; @@ -348,14 +346,35 @@ RETURN_CODE UBRing::UbrAddCloseTimer() { } RETURN_CODE UBRing::UbrAddTimer() { - if (BAIDU_UNLIKELY(UbrAddCloseTimer() != UBRING_OK)) { - LOG(ERROR) << "Ubr " << _trx->local_shm.name << " add closed timer failed."; - return UBRING_ERR; - } - - if (BAIDU_UNLIKELY(UbrAddHBTimer() != UBRING_OK)) { + // Arm both timers with the manager lock held and only while the manager is + // not shutting down: UbrMgrFini then knows that no per-trx timer can appear + // after it took the shutdown flag, so waiting once for the timers it + // snapshotted is enough. Nothing here may block on a timer callback -- the + // callbacks take the manager lock themselves. + bool armed_close = false; + bool armed_hb = false; + const bool allowed = UBRingManager::ArmTimersExclusive( + [this, &armed_close, &armed_hb]() { + armed_close = (UbrAddCloseTimer() == UBRING_OK); + if (armed_close) { + armed_hb = (UbrAddHBTimer() == UBRING_OK); + } + return armed_close && armed_hb; + }); + if (BAIDU_UNLIKELY(!allowed || !armed_close || !armed_hb)) { + // Disarm outside the arming exclusion: waiting for a dispatched + // callback while holding the manager lock would deadlock against the + // callbacks that take it. UbrTimerDelAndWait(&_trx->close_timer); - LOG(ERROR) << "Ubr " << _trx->local_shm.name << " add heartbeat timer failed."; + UbrTimerDelAndWait(&_trx->hb_timer); + if (!allowed) { + // A shutdown refusal is expected, not a failure to report as one. + LOG(WARNING) << "Ubr " << _trx->local_shm.name + << " timer not armed, ubr manager is shutting down."; + } else { + LOG(ERROR) << "Ubr " << _trx->local_shm.name << " add timer failed, close=" + << armed_close << ", hb=" << armed_hb; + } return UBRING_ERR; } return UBRING_OK; @@ -374,14 +393,21 @@ RETURN_CODE UBRing::UbrAddTimer() { // one. bthread dispatches all timer callbacks from a single global timer // thread, so two per-trx callbacks never run concurrently, and a cleanup that // runs from a one-shot timer callback (the delayed-clear path) is serialized -// after every per-trx callback that was dispatched before it. Teardown that -// happens on any other thread waits for a dispatched callback through -// UbrStopTrxTimer -> UbrTimerDelAndWait before it clears the trx or frees its -// shared memory (UbrClearResourceCheck, UbrPassiveClearTrx), and the force -// close and UbrMgrFini paths do the same. FLAGS_ub_flying_io_timeout_s only -// delays the delayed-clear work so that in-flight IO can drain; the callback -// side is already ordered by the timer thread. The UbrTrx pool itself is freed -// only by UbrMgrFini, which waits for these timers first. +// after every per-trx callback that was dispatched before it. Entering a close +// claims the slot for the generation under g_ubr_trx_mgr_mtx +// (UBRingManager::TryClaimTrxClose), so a caller working from a stale snapshot +// -- the faulty-shm event, which looks the slot up before the claim -- cannot +// touch the next occupant. Teardown that happens on any other thread then waits +// for a dispatched callback through UbrStopTrxTimer -> UbrTimerDelAndWait +// before it clears the trx or frees its shared memory (UbrClearResourceCheck, +// UbrPassiveClearTrx), and the force close and UbrMgrFini paths do the same. +// That wait also covers a callback that stopped its own periodic timer from +// inside: the timer facade keeps such a task anchored until the callback +// returns precisely so these teardown paths cannot mistake a running callback +// for an idle trx. FLAGS_ub_flying_io_timeout_s only delays the delayed-clear +// work so that in-flight IO can drain; the callback side is already ordered by +// the timer thread. The UbrTrx pool itself is freed only by UbrMgrFini, which +// waits for these timers first. void* UBRing::UbrTrxCloseCallback(void* args, uint64_t gen) { auto* trx = (UbrTrx*) args; // UbrTrxCallbackCheck rejects a null trx (and cleared queues) before we @@ -424,7 +450,7 @@ void* UBRing::UbrTrxCloseCallback(void* args, uint64_t gen) { break; } remote_rx_event_q->flag = UBR_STATE_CLOSED; - RETURN_CODE clear_rc = ClearTrxResource(trx, gen, true); + RETURN_CODE clear_rc = ClearTrxResource(trx, gen); if (BAIDU_UNLIKELY(clear_rc != UBRING_OK && clear_rc != UBRING_REENTRY)) { LOG(ERROR) << "Trx close callback failed, " << trx->local_shm.name << " clear trx resource failed."; break; @@ -439,10 +465,19 @@ RETURN_CODE UBRing::UbrAddHBTimer() { return UBRING_ERR; } + // A zero or negative ub_hb_timer_interval_s would make the heartbeat fire + // once (or, after the unsigned conversion, effectively never), so the link + // would silently lose its liveness detection. UbrTimerStartPeriodic + // rejects the zero interval; reject the negative one here as well. + if (BAIDU_UNLIKELY(FLAGS_ub_hb_timer_interval_s <= 0)) { + LOG(ERROR) << "Start ubr heartbeat timer failed, ub_hb_timer_interval_s=" + << FLAGS_ub_hb_timer_interval_s << " must be positive."; + return UBRING_ERR; + } const uint64_t interval_us = (uint64_t)FLAGS_ub_hb_timer_interval_s * SEC_TO_USEC; - RETURN_CODE rc = UbrTimerStart(&_trx->hb_timer, 0, interval_us, - UbrTrxHBCallback, (void*)_trx, - ATOMIC_LOAD(_trx->ubr_id)); + RETURN_CODE rc = UbrTimerStartPeriodic(&_trx->hb_timer, 0, interval_us, + UbrTrxHBCallback, (void*)_trx, + ATOMIC_LOAD(_trx->ubr_id)); if (BAIDU_UNLIKELY(rc != UBRING_OK)) { LOG(ERROR) << "Start ubr heartbeat timer failed."; return UBRING_ERR; @@ -450,20 +485,22 @@ RETURN_CODE UBRing::UbrAddHBTimer() { return UBRING_OK; } -RETURN_CODE UBRing::UbrPassiveClearTrx(UbrTrx *trx, uint64_t expect_ubr_id, - bool in_timer_callback) { - RETURN_CODE passive_close_check_rc = UbrTrxCloseCheck(trx); +RETURN_CODE UBRing::UbrPassiveClearTrx(UbrTrx *trx, uint64_t expect_ubr_id) { + RETURN_CODE passive_close_check_rc = UbrTrxCloseCheck(trx, expect_ubr_id); if (BAIDU_UNLIKELY(passive_close_check_rc != UBRING_OK)) { if (passive_close_check_rc == UBRING_REENTRY) { LOG(INFO) << "Passive close skipped, active close in progress, name=" << trx->local_shm.name; - return ClearTrxResource(trx, expect_ubr_id, in_timer_callback); + return ClearTrxResource(trx, expect_ubr_id); } + // UBRING_ERR means this generation no longer owns the slot: the event + // belongs to a trx that was released (and possibly reused) meanwhile, + // so its state and timers must not be touched at all. return UBRING_ERR; } trx->ubr_tx.trx_state = UBR_STATE_CLOSED; trx->ubr_rx.trx_state = UBR_STATE_CLOSED; - UbrStopTrxTimer(&trx->close_timer, in_timer_callback); - UbrStopTrxTimer(&trx->hb_timer, in_timer_callback); + UbrStopTrxTimer(&trx->close_timer); + UbrStopTrxTimer(&trx->hb_timer); // Wait for in-flight IO on a one-shot timer instead of sleeping on the // timer thread. return UbrScheduleClearTimer(trx, expect_ubr_id, @@ -532,7 +569,7 @@ void* UBRing::UbrTrxHBCallback(void* args, uint64_t gen) { int fd = (int)trx->local_shm.fd; LOG(INFO) << "Ubr heartbeat, start to clear trx resource. shm_fd=" << fd << ", shm_name=" << trx->local_shm.name; - UbrPassiveClearTrx(trx, gen, true); + UbrPassiveClearTrx(trx, gen); LOG(INFO) << "Ubr heartbeat clear trx resource finish."; return nullptr; } @@ -1136,6 +1173,9 @@ RETURN_CODE UBRing::UbrMapRemoteShmAddTimer(SHM *local_trx_shm, const char *loca if (BAIDU_UNLIKELY(UbrAddTimer() != UBRING_OK)) { LOG(ERROR) << "Ubr add timer failed, local_name=" << local_name; + // The trx slot stays acquired on purpose: the endpoint keeps this + // UBRing (it falls back to TCP) and returns the slot through its own + // Reset()/UbrTrxClose path. Releasing it here would race that owner. ShmRemoteFree(&_trx->remote_shm); return UBRING_ERR; } @@ -1147,6 +1187,8 @@ RETURN_CODE UBRing::UbrMapRemoteShmAddTimer(SHM *local_trx_shm, const char *loca LOG(ERROR) << "Local shm " << local_trx_shm->name << " wait for connect remote map timeout."; UbrTimerDelAndWait(&_trx->hb_timer); UbrTimerDelAndWait(&_trx->close_timer); + // Same ownership as above: the slot is returned by the endpoint's + // close path, not here. ShmRemoteFree(&_trx->remote_shm); return UBRING_ERR_TIMEOUT; } @@ -1237,7 +1279,7 @@ RETURN_CODE UBRing::WritevHasEnoughSpace(size_t buf_len) return UBRING_OK; } -RETURN_CODE UBRing::UbrClearResourceCheck(UbrTrx *trx, bool in_timer_callback) +RETURN_CODE UBRing::UbrClearResourceCheck(UbrTrx *trx) { if (BAIDU_UNLIKELY(trx == nullptr)) { LOG(ERROR) << "Trx close failed, trx is null."; @@ -1253,10 +1295,11 @@ RETURN_CODE UBRing::UbrClearResourceCheck(UbrTrx *trx, bool in_timer_callback) local_tx_event_q->flag = UBR_STATE_CLOSING; } - // Wait out a dispatched callback before the slot's trx is cleared; inside - // the callback itself only the non-blocking form is possible. - UbrStopTrxTimer(&trx->close_timer, in_timer_callback); - UbrStopTrxTimer(&trx->hb_timer, in_timer_callback); + // Wait out a dispatched callback before the slot's trx is cleared. When + // this runs inside a per-trx callback, UbrTimerDelAndWait recognizes its own + // timer and degrades to the non-blocking delete. + UbrStopTrxTimer(&trx->close_timer); + UbrStopTrxTimer(&trx->hb_timer); if (local_tx_event_q->flag == UBR_STATE_CLOSING) { local_tx_event_q->flag = UBR_STATE_CLOSED; @@ -1266,10 +1309,9 @@ RETURN_CODE UBRing::UbrClearResourceCheck(UbrTrx *trx, bool in_timer_callback) return UBRING_OK; } -RETURN_CODE UBRing::ClearTrxResource(UbrTrx *trx, uint64_t expect_ubr_id, - bool in_timer_callback) +RETURN_CODE UBRing::ClearTrxResource(UbrTrx *trx, uint64_t expect_ubr_id) { - RETURN_CODE rc = UbrClearResourceCheck(trx, in_timer_callback); + RETURN_CODE rc = UbrClearResourceCheck(trx); if (rc != UBRING_OK) { return rc; } @@ -1283,23 +1325,21 @@ RETURN_CODE UBRing::ClearTrxResource(UbrTrx *trx, uint64_t expect_ubr_id, return UBRING_OK; } -RETURN_CODE UBRing::UbrTrxCloseCheck(UbrTrx *trx) +RETURN_CODE UBRing::UbrTrxCloseCheck(UbrTrx *trx, uint64_t expect_ubr_id) { if (BAIDU_UNLIKELY(trx == nullptr)) { LOG(ERROR) << "Trx close failed, client trx is null."; return UBRING_ERR; } - int expected = MAX_CLOSE_COUNT; - if (!ATOMIC_COMPARE_EXCHANGE_STRONG(trx->close_cnt, expected, MAX_CLOSE_COUNT - 1)) { + // Validate the generation and claim the close in one critical section: + // checking here and claiming afterwards would let a slot released and + // reused in between be closed as if it were still this generation's. + RETURN_CODE rc = UBRingManager::TryClaimTrxClose(trx->trx_mgr_index, + expect_ubr_id); + if (rc == UBRING_REENTRY) { LOG(INFO) << "Trx close skipped, already closing, trx local name=" << trx->local_shm.name; - return UBRING_REENTRY; } - - if (BAIDU_UNLIKELY(trx->ubr_tx.local_tx_event_q.addr == nullptr)) { - LOG(ERROR) << "Trx close failed, local_tx_event_q addr is NULL, trx local name=" << trx->local_shm.name; - return UBRING_ERR; - } - return UBRING_OK; + return rc; } ssize_t UBRing::StartReadv(UbrTrx *trx, const struct iovec *iov, int iovcnt, size_t remain_buf_len) diff --git a/src/brpc/ubshm/ub_ring.h b/src/brpc/ubshm/ub_ring.h index 1f720c54ad..0fb440700a 100644 --- a/src/brpc/ubshm/ub_ring.h +++ b/src/brpc/ubshm/ub_ring.h @@ -45,23 +45,20 @@ class UBRing : public butil::IReader { RETURN_CODE UbrTrxClose(); - RETURN_CODE UbrAddCloseTimer(); - + // Arms both per-trx timers. The only entry point for them: it holds the + // manager's arming exclusion, which UbrMgrFini relies on to know that no + // new timer can appear after it took the shutdown flag. RETURN_CODE UbrAddTimer(); static void *UbrTrxCloseCallback(void *args, uint64_t gen); - RETURN_CODE UbrAddHBTimer(); - static void *UbrTrxHBCallback(void *args, uint64_t gen); - // `in_timer_callback' is true only when called from UbrTrxCloseCallback or - // UbrTrxHBCallback, i.e. from a per-trx timer callback: the teardown then - // cannot wait for a dispatched sibling callback (that would join the - // running task), and does not need to, because bthread runs all timer - // callbacks on one global timer thread. - static RETURN_CODE UbrPassiveClearTrx(UbrTrx *trx, uint64_t expect_ubr_id, - bool in_timer_callback); + // Teardown entry point for a per-trx timer callback and for the faulty-shm + // event. `expect_ubr_id' is the generation the caller saw; the close is + // claimed atomically for that generation, so a slot released and reused in + // between is left alone. + static RETURN_CODE UbrPassiveClearTrx(UbrTrx *trx, uint64_t expect_ubr_id); static RETURN_CODE UbrAddAsynClearTimer(UbrTrx *trx, uint64_t expect_ubr_id); @@ -210,19 +207,23 @@ class UBRing : public butil::IReader { } private: + // Only UbrAddTimer may call these: arming outside its manager-held + // exclusion would break the UbrMgrFini invariant described above. + RETURN_CODE UbrAddCloseTimer(); + RETURN_CODE UbrAddHBTimer(); + RETURN_CODE UbrTrxMapLocalShm(SHM *local_shm); RETURN_CODE UbrTrxMapRemoteShm(SHM *remote_shm); RETURN_CODE ApplyAndMapLocalShm(SHM *local_trx_shm, const char *local_name); RETURN_CODE ApplyAndMapRemoteShm(SHM *remote_trx_shm); - static RETURN_CODE UbrTrxCloseCheck(UbrTrx *trx); + static RETURN_CODE UbrTrxCloseCheck(UbrTrx *trx, uint64_t expect_ubr_id); void ReleaseFileLock(int lock_fd); ssize_t StartReadv(UbrTrx *trx, const struct iovec *iov, int iovcnt, size_t remain_buf_len); void PreWriteAddr(uint8_t *addr, size_t len); RETURN_CODE WritevHasEnoughSpace(size_t buf_len); RETURN_CODE UbrServerTrxInit(SHM *local_shm, SHM *remote_shm); - static RETURN_CODE UbrClearResourceCheck(UbrTrx *trx, bool in_timer_callback); - static RETURN_CODE ClearTrxResource(UbrTrx *trx, uint64_t expect_ubr_id, - bool in_timer_callback); + static RETURN_CODE UbrClearResourceCheck(UbrTrx *trx); + static RETURN_CODE ClearTrxResource(UbrTrx *trx, uint64_t expect_ubr_id); UbrTrx* _trx{nullptr}; }; diff --git a/src/brpc/ubshm/ub_ring_manager.cpp b/src/brpc/ubshm/ub_ring_manager.cpp index 44e7024409..89529e37e6 100644 --- a/src/brpc/ubshm/ub_ring_manager.cpp +++ b/src/brpc/ubshm/ub_ring_manager.cpp @@ -60,6 +60,8 @@ RETURN_CODE UBRingManager::UbrMgrDefault() { g_ubr_mgr.trx_num = 0; g_ubr_mgr.trx_cap = FLAGS_ubr_max_managed_num; + g_ubr_mgr.shutting_down = false; + g_ubr_mgr.active_pool_ops = 0; g_ubr_mgr.trx_mgr_unit_status = nullptr; g_ubr_mgr.trx_mgr = nullptr; g_ubr_mgr.trx_mgr_unit_id = nullptr; @@ -91,7 +93,13 @@ RETURN_CODE UBRingManager::UbrMgrInit() { return UBRING_ERR; } - memset(g_ubr_mgr.trx_mgr, 0, trx_mgr_size); + // UbrTrx holds butil::atomic members, so it is not trivially copyable and + // must not be memset: value-initialize every slot instead. A slot is + // re-initialized by AcquireUbrTrxFromMgr before it is handed out, and no + // code reads a slot that was never acquired. + for (uint32_t i = 0; i < g_ubr_mgr.trx_cap; ++i) { + new (&g_ubr_mgr.trx_mgr[i]) UbrTrx(); + } memset(g_ubr_mgr.trx_mgr_unit_status, UBR_MGR_UNIT_FREE, trx_mgr_status_size); memset(g_ubr_mgr.trx_mgr_unit_id, 0, trx_mgr_id_size); memset(g_ubr_mgr.trx_mgr_unit_ctl, 0, trx_mgr_ctl_size); @@ -100,6 +108,14 @@ RETURN_CODE UBRingManager::UbrMgrInit() { } void UBRingManager::UbrMgrFini() { + // Refuse new pool users and new acquisitions first, unconditionally: the + // partial-init path below (a failed allocation) must not let a callback + // walk arrays that the failure left null either. + { + BAIDU_SCOPED_LOCK(g_ubr_trx_mgr_mtx); + g_ubr_mgr.shutting_down = true; + } + // The pool arrays are only safe to walk once UbrMgrInit has allocated and // zeroed all four of them; its allocation-failure path calls this function // with some of them still null and the others uninitialized. @@ -125,6 +141,25 @@ void UBRingManager::UbrMgrFini() { } } } + // Wait for the callback-driven pool accesses that are already running + // (they read UbrTrx without owning a timer), then for the timers. Both + // waits must happen without g_ubr_trx_mgr_mtx: the close/heartbeat + // callbacks and the accesses themselves take that lock. + for (;;) { + { + BAIDU_SCOPED_LOCK(g_ubr_trx_mgr_mtx); + if (g_ubr_mgr.active_pool_ops == 0) { + break; + } + } + usleep(1000); + } + // No timer can be armed after the flag: arming and taking the flag are + // serialized by the manager lock (ArmTimersExclusive), and a cleanup + // control object can no longer be published either + // (TryPublishUnitCleanupCtl refuses once shutting down). So the timers + // armed before the flag are the complete set, and stopping them all + // leaves the pool quiescent. for (uint32_t i : used_slots) { UbrTimerDelAndWait(&g_ubr_mgr.trx_mgr[i].close_timer); UbrTimerDelAndWait(&g_ubr_mgr.trx_mgr[i].hb_timer); @@ -196,6 +231,10 @@ RETURN_CODE UBRingManager::AcquireUbrTrxFromMgr(UbrTrx **trx) { } BAIDU_SCOPED_LOCK(g_ubr_trx_mgr_mtx); + if (BAIDU_UNLIKELY(g_ubr_mgr.shutting_down)) { + LOG(ERROR) << "Acquire trx failed, ubr manager is shutting down."; + return UBRING_ERR; + } if (g_ubr_mgr.trx_num >= g_ubr_mgr.trx_cap) { LOG(ERROR) << "Acquire trx failed, trx number is full."; return UBRING_ERR; @@ -313,18 +352,89 @@ bool UBRingManager::IsUbrTrxSlotUsed(uint32_t idx, uint64_t expect_ubr_id) { g_ubr_mgr.trx_mgr_unit_id[idx] == expect_ubr_id; } +bool UBRingManager::BeginPoolAccess() { + BAIDU_SCOPED_LOCK(g_ubr_trx_mgr_mtx); + // Check every array the caller will walk, not just trx_mgr: UbrMgrInit can + // fail after allocating some of them, and that path calls UbrMgrFini with + // pool_ready == false. + if (BAIDU_UNLIKELY(g_ubr_mgr.shutting_down || + g_ubr_mgr.trx_mgr == nullptr || + g_ubr_mgr.trx_mgr_unit_status == nullptr || + g_ubr_mgr.trx_mgr_unit_id == nullptr || + g_ubr_mgr.trx_mgr_unit_ctl == nullptr)) { + return false; + } + ++g_ubr_mgr.active_pool_ops; + return true; +} + +void UBRingManager::FinishPoolAccess() { + BAIDU_SCOPED_LOCK(g_ubr_trx_mgr_mtx); + if (g_ubr_mgr.active_pool_ops > 0) { + --g_ubr_mgr.active_pool_ops; + } +} + +bool UBRingManager::ArmTimersExclusive(const std::function& arm) { + BAIDU_SCOPED_LOCK(g_ubr_trx_mgr_mtx); + if (BAIDU_UNLIKELY(g_ubr_mgr.shutting_down)) { + return false; + } + return arm(); +} + +RETURN_CODE UBRingManager::TryClaimTrxClose(uint32_t idx, uint64_t expect_ubr_id) { + BAIDU_SCOPED_LOCK(g_ubr_trx_mgr_mtx); + if (BAIDU_UNLIKELY(g_ubr_mgr.trx_mgr == nullptr || + g_ubr_mgr.trx_mgr_unit_status == nullptr || + g_ubr_mgr.trx_mgr_unit_id == nullptr || + idx >= g_ubr_mgr.trx_cap)) { + return UBRING_ERR; + } + // A caller that snapshotted the slot before a release/reuse must not claim + // the close of the slot's new occupant: its generation check and the claim + // happen together here, under the lock the acquire/release paths also take. + if (g_ubr_mgr.trx_mgr_unit_status[idx] != UBR_MGR_UNIT_USED || + g_ubr_mgr.trx_mgr_unit_id[idx] != expect_ubr_id) { + return UBRING_ERR; + } + UbrTrx* trx = &g_ubr_mgr.trx_mgr[idx]; + // Validate before claiming, so a rejected close leaves the counter intact + // and the slot is not stuck in "closing" forever. + if (BAIDU_UNLIKELY(trx->ubr_tx.local_tx_event_q.addr == nullptr)) { + LOG(ERROR) << "Trx close failed, local_tx_event_q addr is NULL, trx local name=" + << trx->local_shm.name; + return UBRING_ERR; + } + int expected = MAX_CLOSE_COUNT; + if (!ATOMIC_COMPARE_EXCHANGE_STRONG(trx->close_cnt, expected, MAX_CLOSE_COUNT - 1)) { + return UBRING_REENTRY; + } + return UBRING_OK; +} + bool UBRingManager::TryPublishUnitCleanupCtl(uint32_t idx, uint64_t expect_ubr_id, UbrCleanupCtl *ctl) { BAIDU_SCOPED_LOCK(g_ubr_trx_mgr_mtx); - if (BAIDU_UNLIKELY(g_ubr_mgr.trx_mgr_unit_ctl == nullptr || + if (BAIDU_UNLIKELY(g_ubr_mgr.shutting_down || + g_ubr_mgr.trx_mgr == nullptr || + g_ubr_mgr.trx_mgr_unit_ctl == nullptr || g_ubr_mgr.trx_mgr_unit_status == nullptr || g_ubr_mgr.trx_mgr_unit_id == nullptr || idx >= g_ubr_mgr.trx_cap || g_ubr_mgr.trx_mgr_unit_status[idx] != UBR_MGR_UNIT_USED || g_ubr_mgr.trx_mgr_unit_id[idx] != expect_ubr_id || g_ubr_mgr.trx_mgr_unit_ctl[idx] != nullptr)) { - return false; // released / reused / already anchored + return false; // shutting down / released / reused / already anchored + } + // Publish on the trx and anchor it in the pool slot inside one critical + // section: a force close that snapshots the slot (also under this lock) + // can then never observe the trx-side publication without its anchor, nor + // claim the no-ctl cleanup path for a cleanup that is being scheduled. + UbrCleanupCtl* expected = nullptr; + if (!g_ubr_mgr.trx_mgr[idx].cleanup_ctl.compare_exchange_strong(expected, ctl)) { + return false; // another schedule won } ctl->ref.fetch_add(1); // manager anchor reference g_ubr_mgr.trx_mgr_unit_ctl[idx] = ctl; @@ -333,12 +443,18 @@ bool UBRingManager::TryPublishUnitCleanupCtl(uint32_t idx, bool UBRingManager::DetachUnitCleanupCtl(uint32_t idx, UbrCleanupCtl *ctl) { BAIDU_SCOPED_LOCK(g_ubr_trx_mgr_mtx); - if (BAIDU_UNLIKELY(g_ubr_mgr.trx_mgr_unit_ctl == nullptr || + if (BAIDU_UNLIKELY(g_ubr_mgr.trx_mgr == nullptr || + g_ubr_mgr.trx_mgr_unit_ctl == nullptr || idx >= g_ubr_mgr.trx_cap || g_ubr_mgr.trx_mgr_unit_ctl[idx] != ctl)) { return false; } g_ubr_mgr.trx_mgr_unit_ctl[idx] = nullptr; + // Clear the trx-side publication in the same critical section, and only + // while it still belongs to `ctl': a slot reused meanwhile may already + // carry the new occupant's control object. + UbrCleanupCtl* published = ctl; + g_ubr_mgr.trx_mgr[idx].cleanup_ctl.compare_exchange_strong(published, nullptr); ctl->ReleaseRef(); // manager reference return true; } @@ -419,27 +535,50 @@ int32_t UBRingManager::UbEventCallback(const char *shm_name) LOG(ERROR) << "Ub event callback failed, shm name is null."; return UBRING_ERR; } - if (BAIDU_UNLIKELY(g_ubr_mgr.trx_mgr == nullptr)) { - LOG(ERROR) << "Ub event callback failed, trx mgr is null."; + // This callback runs on an SDK thread with no timer or cleanup reference + // keeping the pool alive, so register the access first: it is refused once + // UbrMgrFini started, and UbrMgrFini waits for the accepted accesses before + // it frees the pool. The UbrTrx* below is therefore valid until + // FinishPoolAccess, even though the lookup lock is released in between. + if (!BeginPoolAccess()) { + LOG(WARNING) << "Ub event callback skipped, ubr manager is shutting down. shm_name=" + << shm_name; return UBRING_ERR; } - LOG(INFO) << "Ub event callback is processing. shm_name=" << shm_name; - - for (uint32_t i = 0; i < g_ubr_mgr.trx_cap; ++i) { - if (g_ubr_mgr.trx_mgr_unit_status[i] == UBR_MGR_UNIT_FREE) { - continue; - } - if (strcmp(g_ubr_mgr.trx_mgr[i].local_shm.name, shm_name) == 0 || // the failed link is this trx's local shm - strcmp(g_ubr_mgr.trx_mgr[i].remote_shm.name, shm_name) == 0) { // the failed link is this trx's remote shm - ++g_ub_event_cnt; - int fd = (int)g_ubr_mgr.trx_mgr[i].local_shm.fd; - LOG(WARNING) << "Ub event callback, the fd of the faulty link is " << fd; - const uint64_t expect_ubr_id = ATOMIC_LOAD(g_ubr_mgr.trx_mgr[i].ubr_id); - return UBRing::UbrPassiveClearTrx(&g_ubr_mgr.trx_mgr[i], expect_ubr_id, false); + // Look the faulty link up under the manager lock: the pool walk reads the + // slot status and the shm names, both of which a concurrent acquire or + // release rewrites. The generation is snapshotted in the same critical + // section, and the close claim below re-validates it, so a slot released + // and reused after this lookup cannot be closed by mistake. + uint32_t idx = 0; + uint64_t expect_ubr_id = 0; + int fd = -1; + bool found = false; + { + BAIDU_SCOPED_LOCK(g_ubr_trx_mgr_mtx); + for (uint32_t i = 0; i < g_ubr_mgr.trx_cap; ++i) { + if (g_ubr_mgr.trx_mgr_unit_status[i] == UBR_MGR_UNIT_FREE) { + continue; + } + if (strcmp(g_ubr_mgr.trx_mgr[i].local_shm.name, shm_name) == 0 || // the failed link is this trx's local shm + strcmp(g_ubr_mgr.trx_mgr[i].remote_shm.name, shm_name) == 0) { // the failed link is this trx's remote shm + idx = i; + expect_ubr_id = ATOMIC_LOAD(g_ubr_mgr.trx_mgr[i].ubr_id); + fd = (int)g_ubr_mgr.trx_mgr[i].local_shm.fd; + found = true; + break; + } } } - return UBRING_ERR; + int32_t rc = UBRING_ERR; + if (found) { + ++g_ub_event_cnt; + LOG(WARNING) << "Ub event callback, the fd of the faulty link is " << fd; + rc = UBRing::UbrPassiveClearTrx(&g_ubr_mgr.trx_mgr[idx], expect_ubr_id); + } + FinishPoolAccess(); + return rc; } } } diff --git a/src/brpc/ubshm/ub_ring_manager.h b/src/brpc/ubshm/ub_ring_manager.h index 131a749ef7..298bbf218e 100644 --- a/src/brpc/ubshm/ub_ring_manager.h +++ b/src/brpc/ubshm/ub_ring_manager.h @@ -18,6 +18,8 @@ #ifndef BRPC_UB_RING_MANAGER_H #define BRPC_UB_RING_MANAGER_H +#include + #include "brpc/ubshm/ubr_trx.h" #include "brpc/ubshm/shm/shm_def.h" #include "brpc/ubshm/common/common.h" @@ -32,6 +34,15 @@ typedef enum { typedef struct TagUbrMgr { uint32_t trx_num; uint32_t trx_cap; + // Set under g_ubr_trx_mgr_mtx once UbrMgrFini started: no pool slot may be + // acquired afterwards, so the shutdown path only has to wait for the + // connections that were already in the pool when it took the flag. + bool shutting_down; + // Callback-driven operations that read the pool without owning a timer or + // cleanup reference (the faulty-shm event lookup). Guarded by + // g_ubr_trx_mgr_mtx; UbrMgrFini waits for it to drop to zero before it + // frees the pool. + uint32_t active_pool_ops; UbrTrx *trx_mgr; UbrMgrUnitStatus *trx_mgr_unit_status; uint64_t *trx_mgr_unit_id; @@ -61,6 +72,19 @@ class UBRingManager { static RETURN_CODE UbrMgrInit(); + // Tear the pool down. Contract: no connection may be in setup or teardown + // while this runs. The shutdown barrier below only covers the paths that + // register themselves (pool acquisitions, timer arming, cleanup-control + // publication, and the faulty-shm callback through BeginPoolAccess); + // UbrTrxClose / ReleaseUbrTrxFromMgr and the rest of a connection setup + // keep touching UbrTrx and would race the FREE_PTR at the end. Today this + // is only reachable from the init-failure path and from tests, but a real + // graceful-shutdown caller must drain the connections first (or extend the + // barrier to those paths). + // + // This is terminal: the shutdown flag stays set and the pool stays freed, + // so the manager only becomes usable again through a new UbrMgrInit (which + // clears the flag and reallocates the pool) -- not by calling Fini twice. static void UbrMgrFini(); static RETURN_CODE AcquireUbrTrxFromMgr(UbrTrx **trx); @@ -79,21 +103,52 @@ class UBRingManager { // given generation (not released or reused meanwhile). static bool IsUbrTrxSlotUsed(uint32_t idx, uint64_t expect_ubr_id); - // Atomically (under the manager lock) anchor `ctl' to the pool slot, - // but only while the slot is still used by the given generation and - // carries no other cleanup control object. On success the ctl gains - // the manager anchor reference (released when the anchor is detached - // or the slot is retired); returns false -- leaving the slot and the - // reference counts untouched -- when the trx was released, reused, or - // a cleanup is already anchored. + // Atomically claim the close of the pool slot `idx' for the generation + // `expect_ubr_id': under the manager lock it validates that the slot is + // still used by that generation -- so a snapshot taken before a + // release/reuse cannot claim the next occupant's close -- and then moves + // the close counter from MAX_CLOSE_COUNT to MAX_CLOSE_COUNT - 1 so no + // other path can claim the same close. Returns UBRING_REENTRY when the + // same generation is already closing, UBRING_ERR when the caller must not + // touch the slot (released, reused, or not fully initialized). + static RETURN_CODE TryClaimTrxClose(uint32_t idx, uint64_t expect_ubr_id); + + // Atomically (under the manager lock) publish `ctl' on the pool slot and + // anchor it there, but only while the slot is still used by the given + // generation and carries no other cleanup control object. Both the + // trx-side field and the manager anchor are set in the same critical + // section, so a concurrent force close never observes one without the + // other. On success the ctl gains the manager anchor reference (released + // when the anchor is detached or the slot is retired); returns false -- + // leaving the slot and the reference counts untouched -- when the trx was + // released, reused, or a cleanup is already anchored. static bool TryPublishUnitCleanupCtl(uint32_t idx, uint64_t expect_ubr_id, UbrCleanupCtl *ctl); - // Detach `ctl' from the pool slot if it is still anchored there. - // Returns true when the detach happened (the manager reference was - // released). + // Detach `ctl' from the pool slot if it is still anchored there, clearing + // the trx-side field in the same critical section (and only while it still + // belongs to `ctl'). Returns true when the detach happened (the manager + // reference was released). static bool DetachUnitCleanupCtl(uint32_t idx, UbrCleanupCtl *ctl); + // Register / release a callback-driven pool access that does not own a + // timer or cleanup reference (currently the faulty-shm event lookup). + // BeginPoolAccess fails once UbrMgrFini started, and UbrMgrFini waits for + // the accepted accesses to finish before it frees the pool, so the caller + // may keep using a UbrTrx* it obtained while the access is open. Every + // successful BeginPoolAccess must be paired with FinishPoolAccess, which + // is safe to call after UbrMgrFini returned. + static bool BeginPoolAccess(); + static void FinishPoolAccess(); + + // Run `arm' with the manager lock held, and only while the manager is not + // shutting down: arming a per-trx timer and taking the shutdown flag are + // then serialized, so once UbrMgrFini set the flag no new timer can appear + // behind its wait loop. `arm' must not block on a timer callback (it may + // only start timers); it returns whether everything was armed. Returns + // false when the manager is shutting down, in which case `arm' never ran. + static bool ArmTimersExclusive(const std::function& arm); + static void LinkInfoInit(void); static void LinkInfoFini(void); static void AcquireLinkInfoToMgr(const char* listener_name, UbrTrx *trx); From 6305d2d439094e3c15eb7b78024aa83419988067 Mon Sep 17 00:00:00 2001 From: Sherlock0203 <1297399478@qq.com> Date: Thu, 8 Oct 2026 15:09:01 +0800 Subject: [PATCH 6/9] fix(ubshm): reject a null trx before loading its generation (#63) UbrTrxClose loaded _trx->ubr_id before calling UbrTrxCloseCheck, which is where the null check lived, so that guard became unreachable and the load dereferenced null. The path is reachable: UbrAllocateLocalShm and UbrAllocateServerShm reset _trx to nullptr on their error paths (a failing shm map, acquire or trx init) while the endpoint keeps the non-null _ub_ring, and UBShmEndpoint::DeallocateResources calls UbrTrxClose unconditionally from Reset(), so a connection whose setup failed crashes when its endpoint is destroyed, or when it falls back to TCP. Check _trx first and return UBRING_ERR, which restores the behavior from before the generation argument was added; DeallocateResources ignores the return value. UbrAddTimer takes the same guard: its failure branch reports _trx->local_shm.name before the guards in UbrAddCloseTimer/UbrAddHBTimer can reject a null trx. UbrTrxCloseCheck keeps its own null check: UbrPassiveClearTrx still routes through it. Tested: cmake --build cmake-build-debug --target brpc_ubring_unittest, and ./test/brpc_ubring_unittest (12/12 passed). --- src/brpc/ubshm/ub_ring.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/brpc/ubshm/ub_ring.cpp b/src/brpc/ubshm/ub_ring.cpp index ae1ea56a49..1d551e3315 100644 --- a/src/brpc/ubshm/ub_ring.cpp +++ b/src/brpc/ubshm/ub_ring.cpp @@ -196,6 +196,15 @@ static RETURN_CODE UbrScheduleClearTimer(UbrTrx *trx, uint64_t expect_ubr_id, } RETURN_CODE UBRing::UbrTrxClose() { + // _trx is still nullptr when the setup failed: UbrAllocateLocalShm and + // UbrAllocateServerShm reset it on their error paths while the endpoint + // keeps the non-null _ub_ring and calls UbrTrxClose from + // DeallocateResources. Reject before the generation load, which would + // dereference the null _trx. + if (BAIDU_UNLIKELY(_trx == nullptr)) { + LOG(ERROR) << "Trx close failed, client trx is null."; + return UBRING_ERR; + } const uint64_t expect_ubr_id = ATOMIC_LOAD(_trx->ubr_id); RETURN_CODE close_check_rc = UbrTrxCloseCheck(_trx, expect_ubr_id); if (BAIDU_UNLIKELY(close_check_rc != UBRING_OK)) { @@ -346,6 +355,14 @@ RETURN_CODE UBRing::UbrAddCloseTimer() { } RETURN_CODE UBRing::UbrAddTimer() { + // The failure branch below reports _trx->local_shm.name, so a null _trx + // would crash before the guards in UbrAddCloseTimer/UbrAddHBTimer get to + // return their error. No current caller reaches this with a null _trx; + // reject it anyway for the same reason those two do. + if (BAIDU_UNLIKELY(_trx == nullptr)) { + LOG(ERROR) << "Trx add timer failed, trx is null."; + return UBRING_ERR; + } // Arm both timers with the manager lock held and only while the manager is // not shutting down: UbrMgrFini then knows that no per-trx timer can appear // after it took the shutdown flag, so waiting once for the timers it From 3dc0a64a24d54ed1908c626b5a32b61404e701dd Mon Sep 17 00:00:00 2001 From: Sherlock0203 <1297399478@qq.com> Date: Fri, 9 Oct 2026 19:19:04 +0800 Subject: [PATCH 7/9] Ubshm transport fix 2 (#72) * fix(ubshm): retire the slot storage before a deleted one-shot timer returns UbrTimerDel wins the handle-slot exchange and reports success for a one-shot timer, but a wrapper that the timer thread already dispatched may still be between its dispatch and its own (losing) slot exchange. The caller of UbrTimerDel treats return 0 as "the callback will not run" and frees the object that stores the slot -- UbrCleanupCtl on the delayed-clear path -- so that late wrapper access is a use-after-free. Track the slot retirement on the task: the one-shot wrapper sets slot_retired right after its slot CAS and before running the user callback, and UbrTimerDel waits for it when bthread_timer_del reports the task as dispatched. Callers that win the cancellation get the same guarantee from the delete itself. The wait only spins on a task-internal flag, so it is safe to do under the trx manager lock that the force-close, pool-acquire and UbrMgrFini callers hold, and it is limited to the one-shot path (a periodic task's slot storage is pool memory that the non-blocking callers never free). * fix(ubshm): claim forced cleanup and cleanup publication atomically The force close used to snapshot the pool slot's cleanup control object under the manager lock and then re-verify "still no ctl" out of the lock. A concurrent SDK-fault callback could publish a delayed cleanup for the same generation in between, so both the force close and the delayed-clear callback would run the cleanup of the same shared memory (double unmap / stale event queues). Replace the two steps with ClaimTrxCleanupForced, which snapshots the ctl and, when there is none, claims the cleanup for the generation inside one critical section, and make TryPublishUnitCleanupCtl refuse to publish once that claim happened (new per-acquisition trx->cleanup_forced flag, reset by the next AcquireUbrTrxFromMgr). Publication and claim are now totally ordered by the manager lock, so exactly one of them owns the cleanup. An existing publication keeps the current ctl->state arbitration untouched. * fix(ubshm): offload ubs sdk cleanup from the global timer thread Timers are bthread based, so every callback in the process runs on one global timer thread. UbsShmCallback and the delayed-clear callbacks still called the UBS SDK there (ubsmem_shmem_unmap/deallocate), and those calls can wait on the SDK daemon with no latency bound, so a slow or failed shm release delayed every other timer in the process -- RPC timeouts, backup requests, bthread_usleep wakeups and ubring's own heartbeat/close checks. Add a lazily started cleanup worker bthread with an MPSC job queue. The timer callbacks now only arbitrate cleanup ownership and post a job (shared-memory drain or trx cleanup); the worker runs the blocking part, and takes over the timer/callback reference of the cleanup control object. Callers run the work inline only when the worker cannot be started. The drain keeps the previous retry semantics (the node stays in the list on unmap failure and the periodic timer retries it) and at most one drain step is in flight, and DestroyShmTimer drains the worker before it frees the list and finalizes the SDK. UbrMgrFini already waits for the control-object references, so it needs no change. * fix(ubshm): serialize cleanup worker posting with the lazy start Post() was not serialized with the lazy start of the worker bthread. Two posters could decide "the worker is started" and "enqueue the job" independently: one thread won the start CAS and was about to see bthread_start_background fail, while another read g_worker_started == true and enqueued a job. Since g_worker_unavailable is terminal and no worker ever runs, that job is never dequeued -- its UbrCleanupCtl reference leaks, the pool slot is never released, and g_cleanup_inflight never returns to zero, so every later UbrCleanupWorker::DrainAndWait (DestroyShmTimer on a teardown path) hangs forever. Take a small mutex in Post() around the unavailable check, the start attempt and the enqueue, so a job can only ever be queued behind a worker that really started. Posting only happens on cleanup paths, the worker never takes the mutex, and a failed start now returns false (callers keep running the work inline) without having enqueued anything. * style(ubshm): prefix the cleanup claim results with UBR_CLEANUP_CLAIM The UbrCleanupClaim members were the only constants in ub_ring_manager.h without the module's UBR_ prefix, next to UBR_MGR_UNIT_* and UBR_CLEANUP_*. Rename them to UBR_CLEANUP_CLAIM_NOT_OURS / _HAS_CTL / _OWNED_NULL. * docs(ubshm): document the cleanup worker and the one-shot delete contract The Timer Management sections still implied all cleanup work runs on the timer thread. Document that the delayed-clear and pending-unmap callbacks only arbitrate ownership and post a job to the cleanup worker, and that a one-shot UbrTimerDel returning 0 now also guarantees the facade is done with the storage the handle slot lives in, so the caller may free the object holding it (UbrCleanupCtl on the delayed-clear path). Also refresh the ordering comment above UbrTrxCloseCallback: with the cleanup body on the worker, the guarantee rests on the generation gate the worker re-applies under the manager lock (IsUbrTrxSlotUsed) plus the per-trx UbrStopTrxTimer/UbrTimerDelAndWait waits, not on the timer thread serializing the cleanup itself. --- docs/cn/ubring.md | 2 +- docs/en/ubring.md | 2 +- src/brpc/ubshm/shm/shm_ubs.cpp | 56 ++++++++-- src/brpc/ubshm/shm/shm_ubs.h | 5 + src/brpc/ubshm/timer/timer_mgr.cpp | 48 ++++++++- src/brpc/ubshm/timer/timer_mgr.h | 25 +++-- src/brpc/ubshm/ub_cleanup_worker.cpp | 153 +++++++++++++++++++++++++++ src/brpc/ubshm/ub_cleanup_worker.h | 91 ++++++++++++++++ src/brpc/ubshm/ub_ring.cpp | 98 +++++++++++------ src/brpc/ubshm/ub_ring_manager.cpp | 43 ++++++-- src/brpc/ubshm/ub_ring_manager.h | 36 ++++++- src/brpc/ubshm/ubr_trx.h | 9 ++ 12 files changed, 499 insertions(+), 69 deletions(-) create mode 100644 src/brpc/ubshm/ub_cleanup_worker.cpp create mode 100644 src/brpc/ubshm/ub_cleanup_worker.h diff --git a/docs/cn/ubring.md b/docs/cn/ubring.md index f4c7c0b841..182de6cba6 100644 --- a/docs/cn/ubring.md +++ b/docs/cn/ubring.md @@ -187,7 +187,7 @@ UBRing 架构包含以下组件： ### 定时器管理 -UBRing 的连接管理和超时处理基于 bthread 定时器（`bthread_timer_add`/`bthread_timer_del`，由 `timer_mgr.cpp` 封装）。非阻塞的 `UbrTimerDel` 可在回调内自删，此时句柄会保留到该回调返回，因此并发的外部 `UbrTimerDelAndWait` 一定能等到它；销毁回调参数所属资源的外部路径用 `UbrTimerDelAndWait` 等待运行中回调退出；一次性定时器触发后自动清理句柄。进入关闭处理时会带上调用方看到的代际，并在 manager 锁内原子认领槽位，因此按旧快照发起的关闭（例如共享内存故障事件回调）不会误伤释放后被复用的新连接。定时器回调运行在进程全局的 bthread 定时线程上，必须快速返回。关闭检查定时器在链路空闲时按指数退避轮询（上限 `ub_event_queue_timer_interval_max_us`，默认 10ms），有流量或正在关闭时恢复快速轮询。 +UBRing 的连接管理和超时处理基于 bthread 定时器（`bthread_timer_add`/`bthread_timer_del`，由 `timer_mgr.cpp` 封装）。非阻塞的 `UbrTimerDel` 可在回调内自删，此时句柄会保留到该回调返回，因此并发的外部 `UbrTimerDelAndWait` 一定能等到它；销毁回调参数所属资源的外部路径用 `UbrTimerDelAndWait` 等待运行中回调退出；一次性定时器触发后自动清理句柄。进入关闭处理时会带上调用方看到的代际，并在 manager 锁内原子认领槽位，因此按旧快照发起的关闭（例如共享内存故障事件回调）不会误伤释放后被复用的新连接。定时器回调运行在进程全局的 bthread 定时线程上，必须快速返回。阻塞的共享内存清理不在这里进行：延迟清理与待回收映射的回调只仲裁清理所有权，并把任务投递给专用的清理 worker bthread（`ub_cleanup_worker.cpp`），由它把 UBS SDK 的 `unmap`/`deallocate` 调用移出定时线程。对一次性定时器，`UbrTimerDel` 返回 0 还额外保证 facade 已不再访问句柄槽位所在的存储（它会等待被定时线程派发但尚未竞争槽位的 wrapper 退出竞争），因此调用方可以释放承载该槽位的对象；延迟清理路径正是据此释放 `UbrCleanupCtl`。关闭检查定时器在链路空闲时按指数退避轮询（上限 `ub_event_queue_timer_interval_max_us`，默认 10ms），有流量或正在关闭时恢复快速轮询。 ## 参考资料 diff --git a/docs/en/ubring.md b/docs/en/ubring.md index 9247360e04..8cb64df6dc 100644 --- a/docs/en/ubring.md +++ b/docs/en/ubring.md @@ -186,7 +186,7 @@ The shared memory manager (`shm_mgr.cpp`) provides a unified interface for diffe ### Timer Management -UBRing connection management and timeout handling are built on bthread timers (`bthread_timer_add`/`bthread_timer_del`, wrapped in `timer_mgr.cpp`). The non-blocking `UbrTimerDel` is safe to call from within a timer callback; a periodic callback that deletes its own timer keeps its handle anchored until the callback returns, so a concurrent `UbrTimerDelAndWait` from a teardown path always waits for it. External teardown that frees resources reachable from the callback argument uses `UbrTimerDelAndWait` to wait out a possibly running callback; one-shot timers clean up their handle automatically when they fire. Entering a close carries the generation the caller saw and claims the pool slot atomically under the manager lock, so a close started from a stale snapshot (the shared-memory fault event, for instance) cannot hit a connection that reused the slot in the meantime. Timer callbacks run on the process-wide bthread timer thread and must return quickly. The close-check timer backs off exponentially while the link is idle (capped by `ub_event_queue_timer_interval_max_us`, 10ms by default) and returns to fast polling once there is traffic or a close in progress. +UBRing connection management and timeout handling are built on bthread timers (`bthread_timer_add`/`bthread_timer_del`, wrapped in `timer_mgr.cpp`). The non-blocking `UbrTimerDel` is safe to call from within a timer callback; a periodic callback that deletes its own timer keeps its handle anchored until the callback returns, so a concurrent `UbrTimerDelAndWait` from a teardown path always waits for it. External teardown that frees resources reachable from the callback argument uses `UbrTimerDelAndWait` to wait out a possibly running callback; one-shot timers clean up their handle automatically when they fire. Entering a close carries the generation the caller saw and claims the pool slot atomically under the manager lock, so a close started from a stale snapshot (the shared-memory fault event, for instance) cannot hit a connection that reused the slot in the meantime. Timer callbacks run on the process-wide bthread timer thread and must return quickly. Blocking shared-memory cleanup is therefore not done there: the delayed-clear and pending-unmap callbacks only settle the cleanup ownership and post a job to a dedicated cleanup worker bthread (`ub_cleanup_worker.cpp`), which runs the UBS SDK `unmap`/`deallocate` calls off the timer thread. For a one-shot timer, a `UbrTimerDel` that returns 0 additionally guarantees that the facade is done with the storage the handle slot lives in -- it waits out a wrapper the timer thread already dispatched -- so the caller may free the object that holds the slot; the delayed-clear path frees `UbrCleanupCtl` that way. The close-check timer backs off exponentially while the link is idle (capped by `ub_event_queue_timer_interval_max_us`, 10ms by default) and returns to fast polling once there is traffic or a close in progress. ## References diff --git a/src/brpc/ubshm/shm/shm_ubs.cpp b/src/brpc/ubshm/shm/shm_ubs.cpp index aa4086619d..699c26dd44 100644 --- a/src/brpc/ubshm/shm/shm_ubs.cpp +++ b/src/brpc/ubshm/shm/shm_ubs.cpp @@ -24,6 +24,7 @@ #include #include "butil/scoped_lock.h" #include "brpc/ubshm/timer/timer_mgr.h" +#include "brpc/ubshm/ub_cleanup_worker.h" #include "brpc/ubshm/common/common.h" #include "brpc/ubshm/shm/shm_def.h" #include "brpc/ubshm/ub_ring_manager.h" @@ -49,6 +50,11 @@ DEFINE_int32(ub_flying_io_timeout_s, 5, char g_region_name[MAX_REGION_NAME_DESC_LENGTH] = {0}; butil::atomic g_shm_timer_id(nullptr); ShmList *g_shm_list = nullptr; +// True while a drain step of g_shm_list is queued on (or running in) the +// cleanup worker. It bounds the job backlog: the periodic timer only posts a +// new step when the previous one finished, and the worker clears the flag at +// the end of every step, successful or not. +static butil::atomic g_shm_drain_pending(false); // Set by UbsShmFini, cleared by UbsShmInit: ShmMgrFini may run more than once, // and the SDK finalize plus the shm list teardown must happen exactly once per // init. @@ -434,13 +440,38 @@ void *UbsShmCallback(void* args, uint64_t) return nullptr; } - // Drain one node per fire and keep the SDK calls outside the lock, so - // a slow daemon cannot stall the timer thread for the whole list. + // The timer thread runs every timer in the process, so it must not call + // into the SDK: post the drain to the cleanup worker. At most one step may + // be in flight, otherwise a slow SDK call would pile up jobs for the same + // list head. + if (g_shm_drain_pending.exchange(true)) { + return nullptr; + } + if (BAIDU_UNLIKELY(!UbrCleanupWorker::PostShmDrain(shm_list, UbsShmDrainNode))) { + // The worker could not be started: keep the pre-worker behaviour and + // drain here rather than leaving the nodes in the list forever. + LOG(ERROR) << "Cleanup worker unavailable, draining shm on the timer thread."; + UbsShmDrainNode(shm_list); + } + return nullptr; +} + +void UbsShmDrainNode(ShmList *shm_list) +{ + if (BAIDU_UNLIKELY(shm_list == nullptr)) { + LOG(ERROR) << "Shm list is null."; + g_shm_drain_pending.store(false); + return; + } + + // Drain one node per step and keep the SDK calls outside the lock, so a + // slow daemon cannot stall other users of the list. SHM shm; { BAIDU_SCOPED_LOCK(shm_list->shm_lock); if (shm_list->head == nullptr) { - return nullptr; + g_shm_drain_pending.store(false); + return; } shm = shm_list->head->shm; } @@ -448,16 +479,18 @@ void *UbsShmCallback(void* args, uint64_t) LOG(ERROR) << "Ubs input shm param is invalid, addr is NULL."; BAIDU_SCOPED_LOCK(shm_list->shm_lock); DeleteShmToList(shm_list); - return nullptr; + g_shm_drain_pending.store(false); + return; } int ret = ubsmem_shmem_unmap(shm.addr, shm.len); if (ret != UBSM_OK) { - if (ret == UBSM_ERR_NET) { - return nullptr; // retried on the next fire + if (ret != UBSM_ERR_NET) { + LOG(ERROR) << "Ubs unmap shm=" << shm.name << " length=" << shm.len << " failed, ret=" << ret; } - LOG(ERROR) << "Ubs unmap shm=" << shm.name << " length=" << shm.len << " failed, ret=" << ret; - return nullptr; // node stays at head, retried + // The node stays at the head: the next timer fire retries it. + g_shm_drain_pending.store(false); + return; } { @@ -469,7 +502,7 @@ void *UbsShmCallback(void* args, uint64_t) if (ret != UBSM_OK) { LOG(ERROR) << "Ubs delete shm=" << shm.name << " failed, ret=" << ret; } - return nullptr; + g_shm_drain_pending.store(false); } RETURN_CODE UbsShmAddTimer(ShmList *shm_list) @@ -529,8 +562,11 @@ RETURN_CODE InitShmTimer(ShmList **shm_list) RETURN_CODE DestroyShmTimer(ShmList **shm_list) { - // Wait out a possibly running UbsShmCallback before tearing shm_list down. + // Wait out a possibly running UbsShmCallback, then the cleanup jobs it + // posted: the worker may be inside the SDK calls of a node that is freed + // right below, and must not race ubsmem_finalize either. UbrTimerDelAndWait(&g_shm_timer_id); + UbrCleanupWorker::DrainAndWait(); if (BAIDU_UNLIKELY(shm_list == nullptr)) { LOG(ERROR) << "Shm list handle is null."; return UBRING_ERR; diff --git a/src/brpc/ubshm/shm/shm_ubs.h b/src/brpc/ubshm/shm/shm_ubs.h index 02ed51cd46..562e1f52c2 100644 --- a/src/brpc/ubshm/shm/shm_ubs.h +++ b/src/brpc/ubshm/shm/shm_ubs.h @@ -42,6 +42,11 @@ RETURN_CODE UbsShmLocalMmap(SHM *shm, int prot); void UbsMemLoggerPrint(int level, const char *msg); void *UbsShmCallback(void* args, uint64_t gen); +// One drain step of the pending-unmap list: unmap the head node and deallocate +// it. Called by the cleanup worker (and inline by UbsShmCallback when that +// worker cannot be started), never by the timer thread directly. The head node +// stays in the list when the unmap fails, so a later timer fire retries it. +void UbsShmDrainNode(ShmList *shm_list); RETURN_CODE UbsShmAddTimer(ShmList *shm_list); RETURN_CODE InitShmTimer(ShmList **shm_list); // Takes the handle so it can be cleared after the list is freed: a caller that diff --git a/src/brpc/ubshm/timer/timer_mgr.cpp b/src/brpc/ubshm/timer/timer_mgr.cpp index 344c365a9f..8eb864dd00 100644 --- a/src/brpc/ubshm/timer/timer_mgr.cpp +++ b/src/brpc/ubshm/timer/timer_mgr.cpp @@ -71,6 +71,14 @@ struct UbrTimerTask { butil::atomic ref; butil::atomic join_pending; // a DelAndWait is waiting butil::atomic done; // refs hit zero, joiner frees + // Set by whoever settles the handle slot: the one-shot wrapper (right + // after its slot CAS, before it runs the user callback), a deleter whose + // bthread_timer_del cancelled the task before dispatch, or the kDead + // path. Once true, no facade code -- including the dispatched wrapper -- + // will ever touch `*slot` again, so the object storing the slot may be + // freed. The scheduled wrapper is kept alive by its own schedule + // reference while a deleter waits on this flag. + butil::atomic slot_retired; }; namespace { @@ -160,6 +168,13 @@ void UbrTimerOnFire(void* p) { // the slot is the single arbiter. UbrTimerId expected = task; const bool owned = task->slot->compare_exchange_strong(expected, nullptr); + // The slot storage is now retired: whatever the competition above + // decided, this wrapper will not dereference `slot' again. A one-shot + // UbrTimerDel that won the exchange (or that cancelled a task which had + // not been dispatched at all) is waiting for this store before it lets + // the caller free the storage, so it must happen before the potentially + // long user callback. + task->slot_retired.store(true); if (owned) { task->cb(task->arg, task->gen); } @@ -208,6 +223,7 @@ RETURN_CODE TimerStartInternal(butil::atomic* slot, uint64_t delay_u task->ref.store(3); // owner + schedule + starter task->join_pending.store(false); task->done.store(false); + task->slot_retired.store(false); // Publish the real task before scheduling so a delete or a DelAndWait // racing the start always has an object to act on or wait for. @@ -293,19 +309,41 @@ int UbrTimerDel(butil::atomic* slot) { bthread_usleep(1000); } if (task->state.load() == kDead) { + // The wrapper never ran and never will: this call is the one that + // settles the slot storage. + task->slot_retired.store(true); ReleaseRef(task); // owner; schedule/starter are return 1; // settled by the kDead path } bthread_timer_t id = task->id.load(); if (id != 0 && bthread_timer_del(id) == 0) { + // Cancelled before dispatch: the timer thread will never call the + // wrapper, so nothing else can retire the slot storage. + task->slot_retired.store(true); ReleaseRef(task); // schedule: cancelled before dispatch - } // ==1: dispatched, OnFire (owned==false) - // releases it + } else if (!task->periodic) { + // One-shot already dispatched: the wrapper may be between its dispatch + // (which made bthread_timer_del return 1) and its slot CAS, so it + // still has to touch the storage before it returns. Wait it out here, + // while this caller still owns the task through the owner reference, + // so that returning 0 really means "the storage is not used anymore". + // The wrapper retires the slot before calling any user code and takes + // no lock doing so, hence this wait cannot deadlock, and it is bounded + // by the OS scheduling of the timer thread. A periodic task needs no + // such wait: its slot storage is the pooled UbrTrx, which the callers + // of the non-blocking delete never free (only UbrMgrFini does, after + // UbrTimerDelAndWait), and its retire-slot load happens after the user + // callback -- which may take the very lock this caller holds. + while (!task->slot_retired.load()) { + bthread_usleep(kTimerPollIntervalUs); + } + } // ==1 periodic: dispatched/running; + // OnFire releases the schedule ref ReleaseRef(task); // owner return 0; // This call won the slot competition. For a one-shot timer, - // the callback will not run. For a periodic timer, future - // rearming is stopped, but an already dispatched or running - // callback may still complete. + // the callback will not run and the slot storage is unused. + // For a periodic timer, future rearming is stopped, but an + // already dispatched or running callback may still complete. } void UbrTimerDelAndWait(butil::atomic* slot) { diff --git a/src/brpc/ubshm/timer/timer_mgr.h b/src/brpc/ubshm/timer/timer_mgr.h index b8d3676ab6..1c35a22499 100644 --- a/src/brpc/ubshm/timer/timer_mgr.h +++ b/src/brpc/ubshm/timer/timer_mgr.h @@ -64,16 +64,27 @@ RETURN_CODE UbrTimerStartPeriodic(butil::atomic* slot, UbrTimerBackoffFn backoff = nullptr); // Non-blocking delete, safe to call from inside the timer callback itself. -// This function does not wait for an already running callback and does not -// protect resources reachable from `arg` on its own. +// This function does not wait for an already running user callback and does +// not protect resources reachable from `arg` on its own. // // Returns 0 when this call wins the handle-slot competition. -// - For a one-shot timer, the callback will not run. +// - For a one-shot timer, the callback will not run. This call additionally +// waits out a wrapper that the timer thread already dispatched: such a +// wrapper may have been preempted between its dispatch and the moment it +// competes for the slot, and it still has to touch the slot once (to lose +// that competition) before it can return. When this call returns 0, the +// facade is therefore completely done with the storage `slot` points to, +// and the caller may free the object that stores the slot -- which is what +// the delayed-clear path does with UbrCleanupCtl. The wait only spins on a +// task-internal flag: the wrapper takes no lock and calls no user code +// while retiring the slot, so a caller holding any ubring lock can still +// wait, and no timer callback can be joined from its own dispatch. // - For a periodic timer, future rearming is stopped, but an already -// dispatched or running callback may still execute once more. Callers -// must not reclaim resources reachable from `arg` based on this return -// alone; use UbrTimerDelAndWait when teardown needs to wait for callbacks. -// A return of 0 therefore never means "the handle is free again". +// dispatched or running callback may still execute once more, and the task +// stays anchored in the slot until that callback returns. Callers must not +// reclaim resources reachable from `arg` based on this return alone; use +// UbrTimerDelAndWait when teardown needs to wait for callbacks. A return of +// 0 therefore never means "the handle is free again". // A periodic callback that deletes its own timer falls into the periodic case // as well, but only marks it stopped: the handle slot stays anchored until the // callback returns, so a concurrent UbrTimerDelAndWait on the same slot really diff --git a/src/brpc/ubshm/ub_cleanup_worker.cpp b/src/brpc/ubshm/ub_cleanup_worker.cpp new file mode 100644 index 0000000000..151862aa5c --- /dev/null +++ b/src/brpc/ubshm/ub_cleanup_worker.cpp @@ -0,0 +1,153 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include "butil/atomicops.h" +#include "butil/containers/mpsc_queue.h" +#include "butil/logging.h" +#include "butil/scoped_lock.h" +#include "bthread/bthread.h" +#include "brpc/ubshm/ub_cleanup_worker.h" +#include "brpc/ubshm/ub_ring_manager.h" + +namespace brpc { +namespace ubring { + +namespace { + +// Cleanup latency is tolerated in seconds (it only delays releasing memory of +// an already dead link), so the idle worker polls instead of spinning and the +// module needs no extra wakeup primitive. +constexpr uint64_t kCleanupIdlePollIntervalUs = 100 * 1000; + +butil::atomic g_worker_started(false); +// Set when the worker bthread could not be created. Terminal: posting then +// fails forever, and every caller falls back to running its work inline -- the +// behaviour from before the worker existed. Never cleared, so no job can ever +// be queued behind a worker that will not run. +butil::atomic g_worker_unavailable(false); +// Posted-but-not-finished jobs. A poster increments it before enqueueing and +// the worker decrements it after the job returned, so DrainAndWait can wait +// for quiescence without observing the queue itself. +butil::atomic g_cleanup_inflight(0); +butil::MPSCQueue g_cleanup_queue; +// Serializes the posters (never taken by the worker) around the lazy start and +// the enqueue; see Post(). +pthread_mutex_t g_worker_mtx = PTHREAD_MUTEX_INITIALIZER; + +void RunTrxCleanup(const UbrCleanupJob& job) { + UbrCleanupCtl* ctl = job.ctl; + // The slot may have been released and reused while the job was queued; the + // cleanup body must then be skipped, but the control object still has to + // be settled (exactly as the timer callback did before the offload). + if (UBRingManager::IsUbrTrxSlotUsed(job.trx->trx_mgr_index, job.ubr_id)) { + job.work(job.trx, job.ubr_id); + } + ATOMIC_STORE(ctl->state, UBR_CLEANUP_DONE); + ctl->ReleaseRef(); // timer/callback reference +} + +void* UbrCleanupWorkerMain(void*) { + UbrCleanupJob job; + for (;;) { + if (!g_cleanup_queue.Dequeue(job)) { + bthread_usleep(kCleanupIdlePollIntervalUs); + continue; + } + if (job.type == UbrCleanupJob::SHM_DRAIN) { + job.shm_step(job.shm_list); + } else { + RunTrxCleanup(job); + } + g_cleanup_inflight.fetch_sub(1); + } + return nullptr; +} + +bool Post(const UbrCleanupJob& job) { + // Serialize the posters so that "the worker is started" and "a job was + // enqueued" cannot be decided by different threads at the same time. + // Without this, a poster could observe g_worker_started == true (set by + // the thread that is starting the bthread) and enqueue a job while that + // start is about to fail: g_worker_unavailable is terminal, so the job + // would never be dequeued -- its UbrCleanupCtl reference would leak and + // g_cleanup_inflight would never return to zero, hanging every later + // DrainAndWait on a teardown path. Posting only happens on cleanup paths, + // so the mutex is free in practice. The worker itself never takes it. + BAIDU_SCOPED_LOCK(g_worker_mtx); + if (g_worker_unavailable.load()) { + return false; + } + if (!g_worker_started.load()) { + bthread_attr_t attr = BTHREAD_ATTR_NORMAL; + bthread_attr_set_name(&attr, "UBCleanup"); + bthread_t tid; + if (BAIDU_UNLIKELY(bthread_start_background( + &tid, &attr, UbrCleanupWorkerMain, nullptr) != 0)) { + LOG(ERROR) << "Fail to start the ubring cleanup worker bthread."; + // Terminal: never queue a job behind a worker that cannot run, so + // UbrCleanupWorker::DrainAndWait can never observe an orphaned job. + g_worker_unavailable.store(true); + return false; + } + g_worker_started.store(true); + } + // Increment before enqueueing so a concurrent DrainAndWait can never + // observe quiescence while this job is still on its way to the queue. + g_cleanup_inflight.fetch_add(1); + g_cleanup_queue.Enqueue(job); + return true; +} + +} // namespace + +bool UbrCleanupWorker::PostShmDrain(ShmList* shm_list, + void (*shm_step)(ShmList* shm_list)) { + if (BAIDU_UNLIKELY(shm_list == nullptr || shm_step == nullptr)) { + return false; + } + UbrCleanupJob job; + job.type = UbrCleanupJob::SHM_DRAIN; + job.shm_list = shm_list; + job.shm_step = shm_step; + return Post(job); +} + +bool UbrCleanupWorker::PostTrxCleanup(UbrTrx* trx, uint64_t ubr_id, + void (*work)(UbrTrx*, uint64_t), + UbrCleanupCtl* ctl) { + if (BAIDU_UNLIKELY(trx == nullptr || work == nullptr || ctl == nullptr)) { + return false; + } + UbrCleanupJob job; + job.type = UbrCleanupJob::TRX_CLEANUP; + job.trx = trx; + job.ubr_id = ubr_id; + job.work = work; + job.ctl = ctl; + return Post(job); +} + +void UbrCleanupWorker::DrainAndWait() { + while (g_cleanup_inflight.load() > 0) { + bthread_usleep(1000); + } +} + +} // namespace ubring +} // namespace brpc diff --git a/src/brpc/ubshm/ub_cleanup_worker.h b/src/brpc/ubshm/ub_cleanup_worker.h new file mode 100644 index 0000000000..005d973fc0 --- /dev/null +++ b/src/brpc/ubshm/ub_cleanup_worker.h @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef BRPC_UB_CLEANUP_WORKER_H +#define BRPC_UB_CLEANUP_WORKER_H + +#include +#include "brpc/ubshm/common/common.h" +#include "brpc/ubshm/shm/shm_def.h" + +namespace brpc { +namespace ubring { + +struct TagUbrTrx; +typedef struct TagUbrTrx UbrTrx; +struct UbrCleanupCtl; + +// One unit of ubring cleanup work. +// +// The process-wide bthread timer thread runs every timer in the process (RPC +// timeouts, backup requests, bthread_usleep wakeups, ubring heartbeats and +// close checks), so a callback that calls into the UBS SDK -- which may wait +// on a network round trip to the SDK daemon -- delays all of them. The timer +// callbacks therefore only arbitrate ownership and post a job here; the +// blocking part runs on the dedicated cleanup worker. +struct UbrCleanupJob { + enum Type { + SHM_DRAIN = 0, // release one pending shared-memory unmap + TRX_CLEANUP = 1 // run the delayed cleanup of one trx generation + }; + + Type type = SHM_DRAIN; + + // SHM_DRAIN: `shm_step' performs the blocking SDK calls and settles the + // producer's retry bookkeeping. It is a function pointer so this worker + // stays independent of the shm backend, and so its scheduling can be + // unit-tested without the SDK. + void (*shm_step)(ShmList* shm_list) = nullptr; + ShmList* shm_list = nullptr; + + // TRX_CLEANUP: the job takes over the timer/callback reference the + // scheduling callback held on `ctl' and releases it when done. `work' is + // the cleanup body (used by UbrDoAsynClearWork / UbrDoPassiveClearWork) + // and only runs if the slot still belongs to `ubr_id'. + UbrTrx* trx = nullptr; + uint64_t ubr_id = 0; + void (*work)(UbrTrx* trx, uint64_t ubr_id) = nullptr; + UbrCleanupCtl* ctl = nullptr; +}; + +// A lazily started, process-wide bthread that drains UbrCleanupJob's. It is +// never stopped during a normal run: after a ShmMgrFini cycle the worker only +// needs to be quiescent (DrainAndWait) before the shared memory it may touch +// is finalized, and an idle worker performs no SDK call at all. Posting is +// serialized, so a job can never be queued behind a worker start that failed. +class UbrCleanupWorker { +public: + // Post a shared-memory drain step. Returns false when the worker cannot be + // started, in which case the caller must run `shm_step' inline. + static bool PostShmDrain(ShmList* shm_list, void (*shm_step)(ShmList* shm_list)); + + // Post a delayed trx cleanup, transferring the timer/callback reference of + // `ctl'. Returns false when the worker cannot be started, in which case + // the caller keeps the reference and must run the cleanup inline. + static bool PostTrxCleanup(UbrTrx* trx, uint64_t ubr_id, + void (*work)(UbrTrx* trx, uint64_t ubr_id), + UbrCleanupCtl* ctl); + + // Wait until every posted job finished. Bounded by one in-flight SDK call; + // returns immediately when the worker never started. + static void DrainAndWait(); +}; + +} // namespace ubring +} // namespace brpc + +#endif //BRPC_UB_CLEANUP_WORKER_H diff --git a/src/brpc/ubshm/ub_ring.cpp b/src/brpc/ubshm/ub_ring.cpp index 1d551e3315..98f845702e 100644 --- a/src/brpc/ubshm/ub_ring.cpp +++ b/src/brpc/ubshm/ub_ring.cpp @@ -23,6 +23,7 @@ #include "bthread/bthread.h" #include "butil/logging.h" #include "brpc/ubshm/ub_ring.h" +#include "brpc/ubshm/ub_cleanup_worker.h" #include "brpc/ubshm/ub_ring_manager.h" #include "brpc/ubshm/shm/shm_ipc.h" @@ -159,26 +160,32 @@ static RETURN_CODE UbrScheduleClearTimer(UbrTrx *trx, uint64_t expect_ubr_id, expect_ubr_id); if (BAIDU_UNLIKELY(rc != UBRING_OK)) { // The timer was never scheduled: this path owns the manager, - // timer/callback and starter references. Roll the schedule back - // and run the cleanup inline so the trx does not end up with - // neither timers nor a queued cleanup. If force close claimed the - // ownership meanwhile, leave the manager anchor to it. + // timer/callback and starter references. Hand the cleanup to the + // worker -- the caller may be the timer thread -- so the trx does not + // end up with neither timers nor a queued cleanup, and run it inline + // only when the worker cannot be started. int state_expected = UBR_CLEANUP_PENDING; if (ATOMIC_COMPARE_EXCHANGE_STRONG(ctl->state, state_expected, UBR_CLEANUP_RUNNING)) { - // Gate on the slot still being used by this generation: force - // close may have claimed the cleanup and released the slot (which - // already freed the trx resources) after we were anchored. - if (UBRingManager::IsUbrTrxSlotUsed(trx->trx_mgr_index, ctl->ubr_id)) { - work(trx, ctl->ubr_id); + if (!UbrCleanupWorker::PostTrxCleanup(trx, ctl->ubr_id, work, ctl)) { + // Gate on the slot still being used by this generation: force + // close may have claimed the cleanup and released the slot + // (which already freed the trx resources) after we were + // anchored. + if (UBRingManager::IsUbrTrxSlotUsed(trx->trx_mgr_index, ctl->ubr_id)) { + work(trx, ctl->ubr_id); + } + ATOMIC_STORE(ctl->state, UBR_CLEANUP_DONE); + ctl->ReleaseRef(); // timer/callback reference } - ATOMIC_STORE(ctl->state, UBR_CLEANUP_DONE); - // Detach only after DONE: while the inline cleanup runs, the - // anchor and cleanup_ctl must keep telling a concurrent force - // close that this cleanup is owned (RUNNING/DONE), otherwise it - // would fall into its no-ctl branch and clean the trx again. - UBRingManager::DetachUnitCleanupCtl(trx->trx_mgr_index, ctl); + // When the job was posted, the worker owns the timer/callback + // reference and settles state/anchor exactly like the timer-fired + // path; the manager anchor keeps the control object alive until + // the slot is acquired again or UbrMgrFini retires it. + } else { + // A force close already owns the cleanup. It could not take the + // reference through the never-armed timer, so release it here. + ctl->ReleaseRef(); // timer/callback reference } - ctl->ReleaseRef(); // timer/callback reference ctl->ReleaseRef(); // starter reference return UBRING_ERR; } @@ -241,13 +248,16 @@ RETURN_CODE UBRing::UbrTrxClose() { // to an already running delayed-clear callback otherwise. UbrTimerDelAndWait(&_trx->close_timer); UbrTimerDelAndWait(&_trx->hb_timer); - UbrCleanupCtl* ctl = UBRingManager::SnapshotUnitCleanupCtl(_trx->trx_mgr_index); - if (ctl != nullptr && ctl->ubr_id != expect_ubr_id) { - ctl->ReleaseRef(); // snapshot reference - ctl = nullptr; // slot reused, not ours - } + // Snapshot the delayed cleanup and, when there is none, claim the + // cleanup for this force close in one critical section. Splitting + // the two would let a concurrent SDK-fault callback publish a new + // cleanup after our empty snapshot, leaving two paths running the + // cleanup of the same shared memory. + UbrCleanupCtl* ctl = nullptr; + const UbrCleanupClaim claim = UBRingManager::ClaimTrxCleanupForced( + _trx->trx_mgr_index, expect_ubr_id, &ctl); bool cleanup_owned = false; - if (ctl != nullptr) { + if (claim == UBR_CLEANUP_CLAIM_HAS_CTL) { int expected = UBR_CLEANUP_PENDING; if (ATOMIC_COMPARE_EXCHANGE_STRONG(ctl->state, expected, UBR_CLEANUP_RUNNING)) { cleanup_owned = true; @@ -255,7 +265,7 @@ RETURN_CODE UBRing::UbrTrxClose() { ctl->ReleaseRef(); // timer/callback reference } } - } else if (ATOMIC_LOAD(_trx->ubr_id) == expect_ubr_id) { + } else if (claim == UBR_CLEANUP_CLAIM_OWNED_NULL) { cleanup_owned = true; } if (cleanup_owned) { @@ -408,10 +418,16 @@ RETURN_CODE UBRing::UbrAddTimer() { // A slot cannot be released and reused underneath a callback that already // passed that check, and the guarantee is a synchronization one, not a timing // one. bthread dispatches all timer callbacks from a single global timer -// thread, so two per-trx callbacks never run concurrently, and a cleanup that -// runs from a one-shot timer callback (the delayed-clear path) is serialized -// after every per-trx callback that was dispatched before it. Entering a close -// claims the slot for the generation under g_ubr_trx_mgr_mtx +// thread, so two per-trx callbacks never run concurrently, and the delayed +// clear arbitration (the ctl->state compare-exchange in the clear callbacks) +// is serialized after every per-trx callback that was dispatched before it. +// The cleanup body itself no longer runs on the timer thread but on the +// cleanup worker (UbrCleanupWorker): it is kept away from a released or reused +// slot by the generation gate it re-applies under g_ubr_trx_mgr_mtx right +// before running the work (IsUbrTrxSlotUsed), and by the scheduling paths that +// wait for the per-trx callbacks through UbrStopTrxTimer -> UbrTimerDelAndWait +// before they may clear the trx or hand its slot on. Entering a close claims +// the slot for the generation under g_ubr_trx_mgr_mtx // (UBRingManager::TryClaimTrxClose), so a caller working from a stale snapshot // -- the faulty-shm event, which looks the slot up before the claim -- cannot // touch the next occupant. Teardown that happens on any other thread then waits @@ -538,12 +554,19 @@ void* UBRing::UbrPassiveClearCallback(void* args, uint64_t) { ctl->ReleaseRef(); return nullptr; } + // The ownership arbitration above is settled; hand the blocking work to + // the cleanup worker instead of running it on the timer thread, which + // serves every timer in the process. The job takes over the timer/callback + // reference. UbrTrx* trx = ctl->trx; - if (BAIDU_UNLIKELY(UBRingManager::IsUbrTrxSlotUsed(trx->trx_mgr_index, ctl->ubr_id))) { - UbrDoPassiveClearWork(trx, ctl->ubr_id); + if (!UbrCleanupWorker::PostTrxCleanup(trx, ctl->ubr_id, + UbrDoPassiveClearWork, ctl)) { + if (BAIDU_UNLIKELY(UBRingManager::IsUbrTrxSlotUsed(trx->trx_mgr_index, ctl->ubr_id))) { + UbrDoPassiveClearWork(trx, ctl->ubr_id); + } + ATOMIC_STORE(ctl->state, UBR_CLEANUP_DONE); + ctl->ReleaseRef(); // timer/callback reference } - ATOMIC_STORE(ctl->state, UBR_CLEANUP_DONE); - ctl->ReleaseRef(); // timer/callback reference return nullptr; } @@ -615,12 +638,17 @@ void *UBRing::UbrAsynClearCallback(void *args, uint64_t) ctl->ReleaseRef(); return nullptr; } + // Same offload as the passive path: the timer thread only arbitrates + // ownership, the worker does the SDK-backed cleanup. UbrTrx* trx = ctl->trx; - if (BAIDU_UNLIKELY(UBRingManager::IsUbrTrxSlotUsed(trx->trx_mgr_index, ctl->ubr_id))) { - UbrDoAsynClearWork(trx, ctl->ubr_id); + if (!UbrCleanupWorker::PostTrxCleanup(trx, ctl->ubr_id, + UbrDoAsynClearWork, ctl)) { + if (BAIDU_UNLIKELY(UBRingManager::IsUbrTrxSlotUsed(trx->trx_mgr_index, ctl->ubr_id))) { + UbrDoAsynClearWork(trx, ctl->ubr_id); + } + ATOMIC_STORE(ctl->state, UBR_CLEANUP_DONE); + ctl->ReleaseRef(); // timer/callback reference } - ATOMIC_STORE(ctl->state, UBR_CLEANUP_DONE); - ctl->ReleaseRef(); // timer/callback reference return nullptr; } diff --git a/src/brpc/ubshm/ub_ring_manager.cpp b/src/brpc/ubshm/ub_ring_manager.cpp index 89529e37e6..c5c654f98b 100644 --- a/src/brpc/ubshm/ub_ring_manager.cpp +++ b/src/brpc/ubshm/ub_ring_manager.cpp @@ -251,6 +251,7 @@ RETURN_CODE UBRingManager::AcquireUbrTrxFromMgr(UbrTrx **trx) { g_ubr_mgr.trx_mgr[i].close_timer = nullptr; g_ubr_mgr.trx_mgr[i].hb_timer = nullptr; g_ubr_mgr.trx_mgr[i].cleanup_ctl = nullptr; + g_ubr_mgr.trx_mgr[i].cleanup_forced.store(false); // Retire the previous acquisition's cleanup control object. UbrCleanupCtl* old_ctl = g_ubr_mgr.trx_mgr_unit_ctl[i]; g_ubr_mgr.trx_mgr_unit_ctl[i] = nullptr; @@ -329,16 +330,45 @@ RETURN_CODE UBRingManager::ReleaseUbrTrxFromMgr(UbrTrx *trx, return UBRING_OK; } -UbrCleanupCtl* UBRingManager::SnapshotUnitCleanupCtl(uint32_t idx) { +UbrCleanupClaim UBRingManager::ClaimTrxCleanupForced( + uint32_t idx, uint64_t expect_ubr_id, UbrCleanupCtl** out_ctl) { + if (out_ctl != nullptr) { + *out_ctl = nullptr; + } BAIDU_SCOPED_LOCK(g_ubr_trx_mgr_mtx); - if (BAIDU_UNLIKELY(g_ubr_mgr.trx_mgr_unit_ctl == nullptr || idx >= g_ubr_mgr.trx_cap)) { - return nullptr; + if (BAIDU_UNLIKELY(g_ubr_mgr.trx_mgr == nullptr || + g_ubr_mgr.trx_mgr_unit_status == nullptr || + g_ubr_mgr.trx_mgr_unit_id == nullptr || + g_ubr_mgr.trx_mgr_unit_ctl == nullptr || + idx >= g_ubr_mgr.trx_cap)) { + return UBR_CLEANUP_CLAIM_NOT_OURS; + } + // Same generation check as TryClaimTrxClose: a caller that snapshotted the + // slot before a release/reuse must not claim the new occupant's cleanup. + if (g_ubr_mgr.trx_mgr_unit_status[idx] != UBR_MGR_UNIT_USED || + g_ubr_mgr.trx_mgr_unit_id[idx] != expect_ubr_id) { + return UBR_CLEANUP_CLAIM_NOT_OURS; } UbrCleanupCtl* ctl = g_ubr_mgr.trx_mgr_unit_ctl[idx]; if (ctl != nullptr) { + // An anchored ctl always belongs to the current generation: the + // publication checked it and the acquire retires the previous one. + // Keep the explicit check as a belt-and-braces guard. + if (BAIDU_UNLIKELY(ctl->ubr_id != expect_ubr_id)) { + return UBR_CLEANUP_CLAIM_NOT_OURS; + } ctl->ref.fetch_add(1); // snapshot reference - } - return ctl; + if (out_ctl != nullptr) { + *out_ctl = ctl; + } + return UBR_CLEANUP_CLAIM_HAS_CTL; + } + // No cleanup was published: this call owns the forced cleanup, and + // refusing later publications keeps it the only owner. TryPublishUnitCleanupCtl + // checks the flag in this same critical section, so a publication either + // happened before (we would have seen its ctl above) or is refused. + g_ubr_mgr.trx_mgr[idx].cleanup_forced.store(true); + return UBR_CLEANUP_CLAIM_OWNED_NULL; } bool UBRingManager::IsUbrTrxSlotUsed(uint32_t idx, uint64_t expect_ubr_id) { @@ -425,8 +455,9 @@ bool UBRingManager::TryPublishUnitCleanupCtl(uint32_t idx, idx >= g_ubr_mgr.trx_cap || g_ubr_mgr.trx_mgr_unit_status[idx] != UBR_MGR_UNIT_USED || g_ubr_mgr.trx_mgr_unit_id[idx] != expect_ubr_id || + g_ubr_mgr.trx_mgr[idx].cleanup_forced.load() || g_ubr_mgr.trx_mgr_unit_ctl[idx] != nullptr)) { - return false; // shutting down / released / reused / already anchored + return false; // shutting down / released / reused / forced / already anchored } // Publish on the trx and anchor it in the pool slot inside one critical // section: a force close that snapshots the slot (also under this lock) diff --git a/src/brpc/ubshm/ub_ring_manager.h b/src/brpc/ubshm/ub_ring_manager.h index 298bbf218e..711c06e339 100644 --- a/src/brpc/ubshm/ub_ring_manager.h +++ b/src/brpc/ubshm/ub_ring_manager.h @@ -31,6 +31,19 @@ typedef enum { UBR_MGR_UNIT_USED = 1 } UbrMgrUnitStatus; +// Result of UBRingManager::ClaimTrxCleanupForced. +typedef enum { + // The slot is not (or no longer) used by the requested generation: the + // caller must not touch it and must not run the cleanup. + UBR_CLEANUP_CLAIM_NOT_OURS = 0, + // A delayed cleanup is published for that generation and was handed back + // in *out_ctl with a snapshot reference the caller must release. + UBR_CLEANUP_CLAIM_HAS_CTL = 1, + // No delayed cleanup existed; the forced claim took the cleanup ownership + // and later publications for that generation are refused. + UBR_CLEANUP_CLAIM_OWNED_NULL = 2 +} UbrCleanupClaim; + typedef struct TagUbrMgr { uint32_t trx_num; uint32_t trx_cap; @@ -95,9 +108,23 @@ class UBRingManager { static RETURN_CODE ReleaseUbrTrxFromMgr(UbrTrx *trx, uint64_t expect_ubr_id); - // Snapshot the cleanup control object of a pool slot. The caller - // receives a reference and must ReleaseRef it on every exit path. - static UbrCleanupCtl* SnapshotUnitCleanupCtl(uint32_t idx); + // Snapshot the delayed cleanup control object of a pool slot and, when + // there is none, claim the cleanup of `expect_ubr_id' for the force close + // that will run it. Both the snapshot and the claim happen under the + // manager lock, i.e. in the same critical section as + // TryPublishUnitCleanupCtl, so the two are totally ordered: either the + // publication ran first (UBR_CLEANUP_CLAIM_HAS_CTL, and the existing + // ctl->state arbitration decides who runs the cleanup) or the claim ran + // first (UBR_CLEANUP_CLAIM_OWNED_NULL, and every later publication for + // this generation is refused through trx->cleanup_forced). Splitting the + // two steps -- a snapshot, then an out-of-lock re-check that no ctl + // appeared -- leaves a window in which a concurrent SDK-fault callback + // publishes a new cleanup after an empty snapshot and both paths run the + // cleanup. `*out_ctl' is only written on UBR_CLEANUP_CLAIM_HAS_CTL, and + // carries a snapshot reference the caller must ReleaseRef. + static UbrCleanupClaim ClaimTrxCleanupForced(uint32_t idx, + uint64_t expect_ubr_id, + UbrCleanupCtl** out_ctl); // Under the manager lock, confirm the pool slot is still used by the // given generation (not released or reused meanwhile). @@ -121,7 +148,8 @@ class UBRingManager { // other. On success the ctl gains the manager anchor reference (released // when the anchor is detached or the slot is retired); returns false -- // leaving the slot and the reference counts untouched -- when the trx was - // released, reused, or a cleanup is already anchored. + // released, reused, a force close already claimed the cleanup, or a + // cleanup is already anchored. static bool TryPublishUnitCleanupCtl(uint32_t idx, uint64_t expect_ubr_id, UbrCleanupCtl *ctl); diff --git a/src/brpc/ubshm/ubr_trx.h b/src/brpc/ubshm/ubr_trx.h index cc35f7223f..5a489117a7 100644 --- a/src/brpc/ubshm/ubr_trx.h +++ b/src/brpc/ubshm/ubr_trx.h @@ -170,6 +170,15 @@ typedef struct TagUbrTrx { butil::atomic close_timer; butil::atomic hb_timer; butil::atomic cleanup_ctl; + // True once a force close claimed the cleanup of this acquisition while no + // delayed cleanup was published (see + // UBRingManager::ClaimTrxCleanupForced). The forced claim and the + // publication of a delayed cleanup are serialized by the manager lock, so + // a later publication sees the flag and refuses: only one of the two ever + // owns the cleanup. Reset by AcquireUbrTrxFromMgr together with the other + // per-acquisition fields; a stale true on a FREE slot is harmless because + // a publication is already refused while the slot is not USED. + butil::atomic cleanup_forced; // Last io ids seen by the close-check timer, used to reset its // back-off polling interval when the link has traffic. uint64_t close_chk_in_io_id; From bb7bf7fb7f9be164176bc61d180ecce257b6dd6a Mon Sep 17 00:00:00 2001 From: Sherlock0203 <1297399478@qq.com> Date: Mon, 12 Oct 2026 17:24:43 +0800 Subject: [PATCH 8/9] Ubshm transport fix 2 (#74) * fix(ubshm): keep the cleanup queue alive past process exit g_cleanup_queue was a file-scope MPSCQueue with a destructor, while the cleanup worker is a bthread that is never stopped: the bthread runtime itself is not destructed at exit (see the note in task_control.cpp), so the worker can still be inside Dequeue() when static destructors run. MPSCQueue's destructor drains the queue, turning that destruction into a second concurrent consumer of a single-consumer queue: _cur_dequeue_node is a plain pointer, so the destructor and the worker could pick and free the same node (double free) or interleave their advances (lost nodes). A timer callback firing during the destructor phase would additionally enqueue into a destroyed queue. No orderly teardown drains the queue at exit -- GlobalRelease only runs on init failure and in UT teardown -- so the residue the destructor would drain is really possible. Allocate the queue with new and never delete it, following the brpc convention for globals entangled with threads that never stop (g_task_control itself); the kernel reclaims it at exit. DrainAndWait observes g_cleanup_inflight, not the queue, so teardown semantics are unchanged. g_cleanup_inflight (trivially destructible atomics) and g_worker_mtx (a static initializer, nothing to destroy) keep their storage. Tested: cmake --build build --target brpc_ubring_unittest, and ./build/test/brpc_ubring_unittest (12/12 passed). * fix(ubshm): retry the cleanup worker start instead of disabling it A failed bthread_start_background in Post() latched g_worker_unavailable forever, so every later cleanup fell back to running inline: the SDK calls went back to the process-wide timer thread (UbsShmDrainNode from UbsShmCallback, the delayed trx cleanups from their callbacks), delaying unrelated RPC timeouts, backup requests and bthread wakeups -- exactly what the worker exists to avoid. bthread creation fails transiently (ENOMEM), so the degradation correlated with memory pressure, the moment where blocking the timer thread hurts most, and it never self-healed. Drop the terminal flag and retry the lazy start on every post made while the worker is not started. A failed attempt leaves g_worker_started false and fails only that post; the caller runs it inline as the safety net, and the next post retries the start. The posting mutex already serializes the start attempt with the enqueue, so a job is still only ever queued behind a start that succeeded -- the flag was redundant double insurance for orphaned jobs. The error is logged with LOG_EVERY_SECOND so a failure window cannot spam. Add the g_ub_test_worker_start_rc hook so the new UT can inject a start failure deterministically (no real bthread is created while it is set), and cover both review cases: repeated failures fail only their own post and queue nothing, and the first post after the injection is cleared retries the start, runs the job exactly once and lets DrainAndWait observe quiescence. The suite must observe the worker in its unstarted state, so it is declared ahead of UBShmEndpointTest, whose teardown posts cleanups, and probes for an already-started worker to skip rather than assert on a start that is never attempted. Tested: cmake --build build --target brpc_ubring_unittest, and ./build/test/brpc_ubring_unittest (13/13 passed). Reintroducing the latch locally makes the new UT fail, confirming it guards the retry. --- src/brpc/ubshm/ub_cleanup_worker.cpp | 74 ++++++++++++++++++---------- src/brpc/ubshm/ub_cleanup_worker.h | 13 +++-- 2 files changed, 57 insertions(+), 30 deletions(-) diff --git a/src/brpc/ubshm/ub_cleanup_worker.cpp b/src/brpc/ubshm/ub_cleanup_worker.cpp index 151862aa5c..c53e351f24 100644 --- a/src/brpc/ubshm/ub_cleanup_worker.cpp +++ b/src/brpc/ubshm/ub_cleanup_worker.cpp @@ -28,6 +28,14 @@ namespace brpc { namespace ubring { +// Test hook, never set in production: when non-zero, the lazy worker start +// inside Post() skips the real bthread_start_background call and behaves as +// if it had returned this value, so a transient start failure (e.g. ENOMEM) +// and its recovery can be exercised deterministically. It is only consulted +// while the worker is not started; read under g_worker_mtx, written by tests +// before posting. +butil::atomic g_ub_test_worker_start_rc(0); + namespace { // Cleanup latency is tolerated in seconds (it only delays releasing memory of @@ -36,16 +44,22 @@ namespace { constexpr uint64_t kCleanupIdlePollIntervalUs = 100 * 1000; butil::atomic g_worker_started(false); -// Set when the worker bthread could not be created. Terminal: posting then -// fails forever, and every caller falls back to running its work inline -- the -// behaviour from before the worker existed. Never cleared, so no job can ever -// be queued behind a worker that will not run. -butil::atomic g_worker_unavailable(false); // Posted-but-not-finished jobs. A poster increments it before enqueueing and // the worker decrements it after the job returned, so DrainAndWait can wait // for quiescence without observing the queue itself. butil::atomic g_cleanup_inflight(0); -butil::MPSCQueue g_cleanup_queue; +// The queue is never destroyed. The cleanup worker bthread outlives every +// orderly teardown -- the bthread runtime itself is not destructed at exit +// (see the note in task_control.cpp), so the worker can still be inside +// Dequeue() while static destructors run. MPSCQueue's destructor drains the +// queue, which would make that destruction a *second* concurrent consumer +// of a single-consumer queue (_cur_dequeue_node is a plain pointer) and +// double-free nodes; a timer callback firing during the destructor phase +// would additionally enqueue into a destroyed queue. Leaking the queue +// follows the brpc convention for globals entangled with threads that +// never stop (g_task_control itself); the kernel reclaims it at exit. +butil::MPSCQueue* const g_cleanup_queue = + new butil::MPSCQueue(); // Serializes the posters (never taken by the worker) around the lazy start and // the enqueue; see Post(). pthread_mutex_t g_worker_mtx = PTHREAD_MUTEX_INITIALIZER; @@ -65,7 +79,7 @@ void RunTrxCleanup(const UbrCleanupJob& job) { void* UbrCleanupWorkerMain(void*) { UbrCleanupJob job; for (;;) { - if (!g_cleanup_queue.Dequeue(job)) { + if (!g_cleanup_queue->Dequeue(job)) { bthread_usleep(kCleanupIdlePollIntervalUs); continue; } @@ -80,29 +94,37 @@ void* UbrCleanupWorkerMain(void*) { } bool Post(const UbrCleanupJob& job) { - // Serialize the posters so that "the worker is started" and "a job was - // enqueued" cannot be decided by different threads at the same time. - // Without this, a poster could observe g_worker_started == true (set by - // the thread that is starting the bthread) and enqueue a job while that - // start is about to fail: g_worker_unavailable is terminal, so the job - // would never be dequeued -- its UbrCleanupCtl reference would leak and - // g_cleanup_inflight would never return to zero, hanging every later - // DrainAndWait on a teardown path. Posting only happens on cleanup paths, - // so the mutex is free in practice. The worker itself never takes it. + // Serialize the posters (the worker itself never takes this mutex) so + // that at most one cleanup worker is ever started: two posters seeing + // "not started" at the same time would start two bthreads -- two + // concurrent consumers of a single-consumer queue. The start attempt and + // the enqueue also share one critical section, so a job is only ever + // queued behind a worker whose start succeeded. Posting only happens on + // cleanup paths, so the mutex is free in practice. BAIDU_SCOPED_LOCK(g_worker_mtx); - if (g_worker_unavailable.load()) { - return false; - } if (!g_worker_started.load()) { + // Retry the lazy start on every post instead of latching a failure: + // bthread_start_background can fail transiently (ENOMEM), and a + // permanent fallback would run every later SDK cleanup inline on the + // global timer thread -- exactly what the worker exists to avoid, + // and worst under the memory pressure that caused the failure. A + // failed attempt leaves g_worker_started false and fails only THIS + // post; the caller runs it inline as the safety net, and the next + // post retries the start. bthread_attr_t attr = BTHREAD_ATTR_NORMAL; bthread_attr_set_name(&attr, "UBCleanup"); bthread_t tid; - if (BAIDU_UNLIKELY(bthread_start_background( - &tid, &attr, UbrCleanupWorkerMain, nullptr) != 0)) { - LOG(ERROR) << "Fail to start the ubring cleanup worker bthread."; - // Terminal: never queue a job behind a worker that cannot run, so - // UbrCleanupWorker::DrainAndWait can never observe an orphaned job. - g_worker_unavailable.store(true); + // A non-zero test hook overrides the start result without creating + // a bthread; see g_ub_test_worker_start_rc. + const int injected_rc = g_ub_test_worker_start_rc.load( + butil::memory_order_relaxed); + const int rc = injected_rc != 0 ? injected_rc + : bthread_start_background(&tid, &attr, + UbrCleanupWorkerMain, nullptr); + if (BAIDU_UNLIKELY(rc != 0)) { + LOG_EVERY_SECOND(ERROR) << "Fail to start the ubring cleanup " + << "worker bthread, running the cleanup " + << "inline."; return false; } g_worker_started.store(true); @@ -110,7 +132,7 @@ bool Post(const UbrCleanupJob& job) { // Increment before enqueueing so a concurrent DrainAndWait can never // observe quiescence while this job is still on its way to the queue. g_cleanup_inflight.fetch_add(1); - g_cleanup_queue.Enqueue(job); + g_cleanup_queue->Enqueue(job); return true; } diff --git a/src/brpc/ubshm/ub_cleanup_worker.h b/src/brpc/ubshm/ub_cleanup_worker.h index 005d973fc0..93e90863ad 100644 --- a/src/brpc/ubshm/ub_cleanup_worker.h +++ b/src/brpc/ubshm/ub_cleanup_worker.h @@ -67,15 +67,20 @@ struct UbrCleanupJob { // needs to be quiescent (DrainAndWait) before the shared memory it may touch // is finalized, and an idle worker performs no SDK call at all. Posting is // serialized, so a job can never be queued behind a worker start that failed. +// A failed start (bthread creation may fail transiently, e.g. ENOMEM) is +// retried on the next post; only the posts made while the start keeps failing +// run inline on the caller. class UbrCleanupWorker { public: - // Post a shared-memory drain step. Returns false when the worker cannot be - // started, in which case the caller must run `shm_step' inline. + // Post a shared-memory drain step. Returns false when the worker could not + // be started for this post (the start is retried on the next post), in + // which case the caller must run `shm_step' inline. static bool PostShmDrain(ShmList* shm_list, void (*shm_step)(ShmList* shm_list)); // Post a delayed trx cleanup, transferring the timer/callback reference of - // `ctl'. Returns false when the worker cannot be started, in which case - // the caller keeps the reference and must run the cleanup inline. + // `ctl'. Returns false when the worker could not be started for this post + // (the start is retried on the next post), in which case the caller keeps + // the reference and must run the cleanup inline. static bool PostTrxCleanup(UbrTrx* trx, uint64_t ubr_id, void (*work)(UbrTrx* trx, uint64_t ubr_id), UbrCleanupCtl* ctl); From 36c2d12e0f56b2a306f66e189d6f399b875d9c95 Mon Sep 17 00:00:00 2001 From: songgaoye Date: Fri, 16 Oct 2026 14:26:48 +0800 Subject: [PATCH 9/9] Validate remote file service names before parsing (#3608) * Validate remote file service names before parsing * Use range-based loop in naming service test --- src/brpc/policy/remote_file_naming_service.cpp | 7 ++++++- test/brpc_naming_service_unittest.cpp | 12 ++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/brpc/policy/remote_file_naming_service.cpp b/src/brpc/policy/remote_file_naming_service.cpp index b2e929544b..a2a83fe4fb 100644 --- a/src/brpc/policy/remote_file_naming_service.cpp +++ b/src/brpc/policy/remote_file_naming_service.cpp @@ -67,7 +67,12 @@ int RemoteFileNamingService::GetServers(const char *service_name_cstr, butil::StringPiece proto; if (pos != butil::StringPiece::npos) { proto = tmpname.substr(0, pos); - for (pos += 3; tmpname[pos] == '/'; ++pos) {} + for (pos += 3; pos < tmpname.size() && tmpname[pos] == '/'; ++pos) {} + if (pos == tmpname.size()) { + LOG(ERROR) << "No server address in service_name=" + << service_name_cstr; + return -1; + } tmpname.remove_prefix(pos); } else { proto = "http"; diff --git a/test/brpc_naming_service_unittest.cpp b/test/brpc_naming_service_unittest.cpp index 2c5ad76d94..a01e033f61 100644 --- a/test/brpc_naming_service_unittest.cpp +++ b/test/brpc_naming_service_unittest.cpp @@ -289,6 +289,18 @@ TEST(NamingServiceTest, remotefile) { } } +TEST(NamingServiceTest, remotefile_invalid_url) { + const char* service_names[] = { + "http://", "bns://", "http:////" + }; + for (const char* service_name : service_names) { + brpc::policy::RemoteFileNamingService rfns; + std::vector servers; + EXPECT_EQ(-1, rfns.GetServers(service_name, &servers)); + EXPECT_TRUE(servers.empty()); + } +} + // Build a JSON array nested `depth' levels deep, e.g. depth=2 -> "[[]]". static std::string MakeDeeplyNestedJson(int depth) { return std::string(depth, '[') + std::string(depth, ']');