From c00f845fcc9cfefcb93761a1afb9f9e7b67a53bf Mon Sep 17 00:00:00 2001 From: Junegunn Choi Date: Mon, 12 Oct 2026 16:35:13 +0900 Subject: [PATCH 1/6] HBASE-30473 Expose Master switch states as numeric metrics - Detect switches left disabled after maintenance. - Avoid quota-table reads during metrics collection. - Distinguish unavailable quota states from disabled states. --- .../hbase/master/MetricsMasterSource.java | 10 ++ .../hbase/master/MetricsMasterSourceImpl.java | 3 + .../hbase/master/MetricsMasterWrapper.java | 6 + .../master/MetricsMasterWrapperImpl.java | 29 ++++ .../hbase/master/TestMasterMetrics.java | 29 ++++ .../hbase/master/TestMasterSwitchMetrics.java | 137 ++++++++++++++++++ .../metrics-and-monitoring.mdx | 31 ++++ 7 files changed, 245 insertions(+) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterSwitchMetrics.java diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java index d606ed630881..1aae8b896bda 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java @@ -71,6 +71,16 @@ public interface MetricsMasterSource extends BaseSource { String CLUSTER_READ_REQUESTS_NAME = "clusterReadRequests"; String CLUSTER_WRITE_REQUESTS_NAME = "clusterWriteRequests"; String OLD_WAL_DIR_SIZE_NAME = "oldWALsDirSize"; + String BALANCER_ENABLED_NAME = "balancerEnabled"; + String NORMALIZER_ENABLED_NAME = "normalizerEnabled"; + String SPLIT_ENABLED_NAME = "splitEnabled"; + String MERGE_ENABLED_NAME = "mergeEnabled"; + String CLEANER_CHORE_ENABLED_NAME = "cleanerChoreEnabled"; + String CATALOG_JANITOR_ENABLED_NAME = "catalogJanitorEnabled"; + String SNAPSHOT_CLEANUP_ENABLED_NAME = "snapshotCleanupEnabled"; + String REPLICATION_PEER_MODIFICATION_ENABLED_NAME = "replicationPeerModificationEnabled"; + String RPC_THROTTLE_ENABLED_NAME = "rpcThrottleEnabled"; + String SWITCH_STATE_DESC = "Switch state: 1 enabled, 0 disabled, -1 unknown"; String MASTER_ACTIVE_TIME_DESC = "Master Active Time"; String MASTER_START_TIME_DESC = "Master Start Time"; String MASTER_FINISHED_INITIALIZATION_TIME_DESC = diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java index 011e66312aa3..d2542efbaecd 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java @@ -132,6 +132,9 @@ public void getMetrics(MetricsCollector metricsCollector, boolean all) { String.valueOf(masterWrapper.getIsActiveMaster())) .addGauge(Interns.info(OLD_WAL_DIR_SIZE_NAME, OLD_WAL_DIR_SIZE_DESC), masterWrapper.getOldWALsDirSize()); + + masterWrapper.getSwitchStates().forEach((name, state) -> metricsRecordBuilder + .addGauge(Interns.info(name, SWITCH_STATE_DESC), state)); } metricsRegistry.snapshot(metricsRecordBuilder, all); diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java index ad8728327014..5ab9035385a1 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java @@ -164,4 +164,10 @@ public interface MetricsMasterWrapper { * Get the size of old WALs directory in bytes. */ long getOldWALsDirSize(); + + /** + * Returns switch states (1 enabled, 0 disabled, -1 unknown), or an empty map unless this Master + * is active and initialized. Implementations must not perform remote reads. + */ + Map getSwitchStates(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java index 67cbbdb8ae9e..417bfeec8171 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java @@ -27,12 +27,14 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.MasterSwitchType; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.quotas.QuotaObserverChore; +import org.apache.hadoop.hbase.quotas.QuotaUtil; import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.PairOfSameType; @@ -267,4 +269,31 @@ public long getOldWALsDirSize() { } return master.getMasterWalManager().getOldWALsDirSize(); } + + @Override + public Map getSwitchStates() { + if (!master.isActiveMaster() || !master.isInitialized()) { + return Collections.emptyMap(); + } + int rpcThrottleEnabled = 0; + if (QuotaUtil.isQuotaEnabled(master.getConfiguration())) { + rpcThrottleEnabled = -1; + if (master.getMasterQuotaManager() != null) { + rpcThrottleEnabled = master.getRpcThrottleStateStore().get() ? 1 : 0; + } + } + return Map.of(MetricsMasterSource.BALANCER_ENABLED_NAME, master.isBalancerOn() ? 1 : 0, + MetricsMasterSource.NORMALIZER_ENABLED_NAME, master.isNormalizerOn() ? 1 : 0, + MetricsMasterSource.SPLIT_ENABLED_NAME, + master.isSplitOrMergeEnabled(MasterSwitchType.SPLIT) ? 1 : 0, + MetricsMasterSource.MERGE_ENABLED_NAME, + master.isSplitOrMergeEnabled(MasterSwitchType.MERGE) ? 1 : 0, + MetricsMasterSource.CLEANER_CHORE_ENABLED_NAME, master.isCleanerChoreEnabled() ? 1 : 0, + MetricsMasterSource.CATALOG_JANITOR_ENABLED_NAME, master.isCatalogJanitorEnabled() ? 1 : 0, + MetricsMasterSource.SNAPSHOT_CLEANUP_ENABLED_NAME, + master.snapshotCleanupStateStore.get() ? 1 : 0, + MetricsMasterSource.REPLICATION_PEER_MODIFICATION_ENABLED_NAME, + master.isReplicationPeerModificationEnabled() ? 1 : 0, + MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, rpcThrottleEnabled); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java index 46cc88825bc3..0eef9a3a4d6a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.StartTestingClusterOption; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.YouAreDeadException; +import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.RegionStatesCount; import org.apache.hadoop.hbase.client.TableDescriptor; @@ -51,6 +52,7 @@ import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.FSTableDescriptors; +import org.apache.hadoop.hbase.util.IOExceptionConsumer; import org.apache.zookeeper.KeeperException; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -147,6 +149,33 @@ public static void after() throws Exception { TEST_UTIL.shutdownMiniCluster(); } + @Test + public void testSwitchMetricsFollowAdminChanges() throws IOException { + Admin admin = TEST_UTIL.getAdmin(); + MetricsMasterSource source = master.getMasterMetrics().getMetricsSource(); + Map> switches = new HashMap<>(); + switches.put("balancerEnabled", enabled -> admin.balancerSwitch(enabled, false)); + switches.put("normalizerEnabled", admin::normalizerSwitch); + switches.put("splitEnabled", enabled -> admin.splitSwitch(enabled, false)); + switches.put("mergeEnabled", enabled -> admin.mergeSwitch(enabled, false)); + switches.put("cleanerChoreEnabled", admin::cleanerChoreSwitch); + switches.put("catalogJanitorEnabled", admin::catalogJanitorSwitch); + switches.put("snapshotCleanupEnabled", enabled -> admin.snapshotCleanupSwitch(enabled, false)); + switches.put("replicationPeerModificationEnabled", admin::replicationPeerModificationSwitch); + Map initialStates = new MetricsMasterWrapperImpl(master).getSwitchStates(); + for (Map.Entry> entry : switches.entrySet()) { + try { + for (boolean enabled : new boolean[] { false, true }) { + entry.getValue().accept(enabled); + metricsHelper.assertGauge(entry.getKey(), enabled ? 1L : 0L, source); + } + } finally { + entry.getValue().accept(initialStates.get(entry.getKey()) == 1); + } + } + metricsHelper.assertGauge("rpcThrottleEnabled", 0L, source); + } + @Test public void testClusterRequests() throws Exception { // sending fake request to master to see how metric value has changed diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterSwitchMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterSwitchMetrics.java new file mode 100644 index 000000000000..5ca7515605ed --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterSwitchMetrics.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CompatibilityFactory; +import org.apache.hadoop.hbase.client.MasterSwitchType; +import org.apache.hadoop.hbase.master.snapshot.SnapshotCleanupStateStore; +import org.apache.hadoop.hbase.metrics.MetricRegistries; +import org.apache.hadoop.hbase.quotas.MasterQuotaManager; +import org.apache.hadoop.hbase.quotas.QuotaUtil; +import org.apache.hadoop.hbase.quotas.RpcThrottleStateStore; +import org.apache.hadoop.hbase.test.MetricsAssertHelper; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) +public class TestMasterSwitchMetrics { + private static final MetricsAssertHelper METRICS_HELPER = + CompatibilityFactory.getInstance(MetricsAssertHelper.class); + + private HMaster master; + private Configuration conf; + private RpcThrottleStateStore rpcThrottleStateStore; + private MetricsMasterWrapperImpl wrapper; + private MetricsMasterSourceImpl source; + + @BeforeEach + public void setUp() { + master = mock(HMaster.class); + conf = new Configuration(false); + conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true); + when(master.getConfiguration()).thenReturn(conf); + when(master.isActiveMaster()).thenReturn(true); + when(master.isInitialized()).thenReturn(true); + when(master.getMasterWalManager()).thenReturn(mock(MasterWalManager.class)); + when(master.getMasterQuotaManager()).thenReturn(mock(MasterQuotaManager.class)); + master.snapshotCleanupStateStore = mock(SnapshotCleanupStateStore.class); + rpcThrottleStateStore = mock(RpcThrottleStateStore.class); + when(master.getRpcThrottleStateStore()).thenReturn(rpcThrottleStateStore); + wrapper = new MetricsMasterWrapperImpl(master); + source = new MetricsMasterSourceImpl(wrapper); + } + + @AfterEach + public void tearDown() { + DefaultMetricsSystem.instance().unregisterSource(MetricsMasterSource.METRICS_JMX_CONTEXT); + MetricRegistries.global().remove(source.getMetricRegistryInfo()); + } + + @Test + public void testEachSwitchChangesItsGauge() { + Map> switches = new HashMap<>(); + switches.put("balancerEnabled", enabled -> when(master.isBalancerOn()).thenReturn(enabled)); + switches.put("normalizerEnabled", enabled -> when(master.isNormalizerOn()).thenReturn(enabled)); + switches.put("splitEnabled", + enabled -> when(master.isSplitOrMergeEnabled(MasterSwitchType.SPLIT)).thenReturn(enabled)); + switches.put("mergeEnabled", + enabled -> when(master.isSplitOrMergeEnabled(MasterSwitchType.MERGE)).thenReturn(enabled)); + switches.put("cleanerChoreEnabled", + enabled -> when(master.isCleanerChoreEnabled()).thenReturn(enabled)); + switches.put("catalogJanitorEnabled", + enabled -> when(master.isCatalogJanitorEnabled()).thenReturn(enabled)); + switches.put("snapshotCleanupEnabled", + enabled -> when(master.snapshotCleanupStateStore.get()).thenReturn(enabled)); + switches.put("replicationPeerModificationEnabled", + enabled -> when(master.isReplicationPeerModificationEnabled()).thenReturn(enabled)); + switches.put("rpcThrottleEnabled", + enabled -> when(rpcThrottleStateStore.get()).thenReturn(enabled)); + Map expected = new HashMap<>(); + switches.keySet().forEach(name -> expected.put(name, 0)); + for (Map.Entry> entry : switches.entrySet()) { + for (boolean enabled : new boolean[] { true, false }) { + entry.getValue().accept(enabled); + expected.put(entry.getKey(), enabled ? 1 : 0); + assertEquals(expected, wrapper.getSwitchStates()); + expected + .forEach((name, state) -> METRICS_HELPER.assertGauge(name, state.longValue(), source)); + } + } + verifyNoInteractions(master.getMasterQuotaManager()); + } + + @Test + public void testOnlyActiveInitializedMasterExposesSwitches() { + Set metricNames = wrapper.getSwitchStates().keySet(); + when(master.isInitialized()).thenReturn(false); + assertTrue(wrapper.getSwitchStates().isEmpty()); + metricNames.forEach(name -> assertFalse(METRICS_HELPER.checkGaugeExists(name, source))); + when(master.isInitialized()).thenReturn(true); + when(master.isActiveMaster()).thenReturn(false); + assertTrue(wrapper.getSwitchStates().isEmpty()); + metricNames.forEach(name -> assertFalse(METRICS_HELPER.checkGaugeExists(name, source))); + } + + @Test + public void testQuotaDisabledAndUnknownStates() { + when(master.getMasterQuotaManager()).thenReturn(null); + assertEquals(-1, wrapper.getSwitchStates().get("rpcThrottleEnabled")); + + conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, false); + when(rpcThrottleStateStore.get()).thenReturn(true); + assertEquals(0, wrapper.getSwitchStates().get("rpcThrottleEnabled")); + } +} diff --git a/hbase-website/app/pages/_docs/docs/_mdx/(multi-page)/operational-management/metrics-and-monitoring.mdx b/hbase-website/app/pages/_docs/docs/_mdx/(multi-page)/operational-management/metrics-and-monitoring.mdx index 88f8957c91fe..55f5bf00417a 100644 --- a/hbase-website/app/pages/_docs/docs/_mdx/(multi-page)/operational-management/metrics-and-monitoring.mdx +++ b/hbase-website/app/pages/_docs/docs/_mdx/(multi-page)/operational-management/metrics-and-monitoring.mdx @@ -133,6 +133,37 @@ The number of regions that have been in transition longer than a threshold time **hbase.master.ritOldestAge** The age of the longest region in transition, in milliseconds +#### Master switch states + +The active, initialized Master exposes numeric switch-state gauges under +`Hadoop:service=HBase,name=Master,sub=Server`. A value of `1` means enabled and +`0` means disabled. A value of `-1` means the state is unknown. The gauges are +absent on backup Masters and before the Master is initialized. +Enabled states indicate that an operation is permitted, rather than that it is +currently running. + +| Gauge | HBase shell command | +| ------------------------------------ | ---------------------------------------------- | +| `balancerEnabled` | `balance_switch true\|false` | +| `normalizerEnabled` | `normalizer_switch true\|false` | +| `splitEnabled` | `splitormerge_switch 'SPLIT', true\|false` | +| `mergeEnabled` | `splitormerge_switch 'MERGE', true\|false` | +| `cleanerChoreEnabled` | `cleaner_chore_switch true\|false` | +| `catalogJanitorEnabled` | `catalogjanitor_switch true\|false` | +| `snapshotCleanupEnabled` | `snapshot_cleanup_switch true\|false` | +| `replicationPeerModificationEnabled` | `peer_modification_switch true\|false` | +| `rpcThrottleEnabled` | `enable_rpc_throttle` / `disable_rpc_throttle` | + +The gauges follow the corresponding enabled-state queries. Maintenance mode +disables the balancer, normalizer, split, and merge states. Per-table settings can +further restrict these operations. The cleaner state covers the default HFile +cleaner and the WAL cleaner. The existing `tag.isBalancerActive` tag under +`Master,sub=Balancer` remains available. + +All switch states are read from memory during metrics collection. +When quota support is disabled, `rpcThrottleEnabled` is `0`. With quota support +enabled, it is `-1` until the quota manager is ready. + ### Most Important RegionServer Metrics Note: Counts are usually over the last metrics reporting interval. From cd37bb7508f886aa6f27e43eed295812602f98b2 Mon Sep 17 00:00:00 2001 From: Junegunn Choi Date: Tue, 13 Oct 2026 10:25:45 +0900 Subject: [PATCH 2/6] HBASE-30473 Use metric constants in switch unit tests --- .../hbase/master/TestMasterSwitchMetrics.java | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterSwitchMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterSwitchMetrics.java index 5ca7515605ed..44a5761374d9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterSwitchMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterSwitchMetrics.java @@ -83,21 +83,23 @@ public void tearDown() { @Test public void testEachSwitchChangesItsGauge() { Map> switches = new HashMap<>(); - switches.put("balancerEnabled", enabled -> when(master.isBalancerOn()).thenReturn(enabled)); - switches.put("normalizerEnabled", enabled -> when(master.isNormalizerOn()).thenReturn(enabled)); - switches.put("splitEnabled", + switches.put(MetricsMasterSource.BALANCER_ENABLED_NAME, + enabled -> when(master.isBalancerOn()).thenReturn(enabled)); + switches.put(MetricsMasterSource.NORMALIZER_ENABLED_NAME, + enabled -> when(master.isNormalizerOn()).thenReturn(enabled)); + switches.put(MetricsMasterSource.SPLIT_ENABLED_NAME, enabled -> when(master.isSplitOrMergeEnabled(MasterSwitchType.SPLIT)).thenReturn(enabled)); - switches.put("mergeEnabled", + switches.put(MetricsMasterSource.MERGE_ENABLED_NAME, enabled -> when(master.isSplitOrMergeEnabled(MasterSwitchType.MERGE)).thenReturn(enabled)); - switches.put("cleanerChoreEnabled", + switches.put(MetricsMasterSource.CLEANER_CHORE_ENABLED_NAME, enabled -> when(master.isCleanerChoreEnabled()).thenReturn(enabled)); - switches.put("catalogJanitorEnabled", + switches.put(MetricsMasterSource.CATALOG_JANITOR_ENABLED_NAME, enabled -> when(master.isCatalogJanitorEnabled()).thenReturn(enabled)); - switches.put("snapshotCleanupEnabled", + switches.put(MetricsMasterSource.SNAPSHOT_CLEANUP_ENABLED_NAME, enabled -> when(master.snapshotCleanupStateStore.get()).thenReturn(enabled)); - switches.put("replicationPeerModificationEnabled", + switches.put(MetricsMasterSource.REPLICATION_PEER_MODIFICATION_ENABLED_NAME, enabled -> when(master.isReplicationPeerModificationEnabled()).thenReturn(enabled)); - switches.put("rpcThrottleEnabled", + switches.put(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, enabled -> when(rpcThrottleStateStore.get()).thenReturn(enabled)); Map expected = new HashMap<>(); switches.keySet().forEach(name -> expected.put(name, 0)); @@ -128,10 +130,10 @@ public void testOnlyActiveInitializedMasterExposesSwitches() { @Test public void testQuotaDisabledAndUnknownStates() { when(master.getMasterQuotaManager()).thenReturn(null); - assertEquals(-1, wrapper.getSwitchStates().get("rpcThrottleEnabled")); + assertEquals(-1, wrapper.getSwitchStates().get(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME)); conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, false); when(rpcThrottleStateStore.get()).thenReturn(true); - assertEquals(0, wrapper.getSwitchStates().get("rpcThrottleEnabled")); + assertEquals(0, wrapper.getSwitchStates().get(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME)); } } From 6d5fe4081f87f12f86dd909a6476abe7b8b04424 Mon Sep 17 00:00:00 2001 From: Junegunn Choi Date: Tue, 13 Oct 2026 10:25:46 +0900 Subject: [PATCH 3/6] HBASE-30473 Explain disabled RPC throttle in metrics test --- .../java/org/apache/hadoop/hbase/master/TestMasterMetrics.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java index 0eef9a3a4d6a..1eaae5113d6a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java @@ -173,6 +173,7 @@ public void testSwitchMetricsFollowAdminChanges() throws IOException { entry.getValue().accept(initialStates.get(entry.getKey()) == 1); } } + // Quota support is disabled by default. metricsHelper.assertGauge("rpcThrottleEnabled", 0L, source); } From 5d9fd6c4cf43439feb29a6499f0c9af3bfba8d89 Mon Sep 17 00:00:00 2001 From: Junegunn Choi Date: Tue, 13 Oct 2026 10:32:26 +0900 Subject: [PATCH 4/6] HBASE-30473 Omit unavailable RPC throttle state Keep switch values boolean during initialization and maintenance mode. --- .../hbase/master/MetricsMasterSource.java | 2 +- .../hbase/master/MetricsMasterWrapper.java | 5 ++- .../master/MetricsMasterWrapperImpl.java | 40 ++++++++++--------- .../hbase/master/TestMasterRepairMode.java | 34 ++++++++++++++++ .../hbase/master/TestMasterSwitchMetrics.java | 22 +++++++++- .../metrics-and-monitoring.mdx | 7 ++-- 6 files changed, 83 insertions(+), 27 deletions(-) diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java index 1aae8b896bda..cf47b8843f1b 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java @@ -80,7 +80,7 @@ public interface MetricsMasterSource extends BaseSource { String SNAPSHOT_CLEANUP_ENABLED_NAME = "snapshotCleanupEnabled"; String REPLICATION_PEER_MODIFICATION_ENABLED_NAME = "replicationPeerModificationEnabled"; String RPC_THROTTLE_ENABLED_NAME = "rpcThrottleEnabled"; - String SWITCH_STATE_DESC = "Switch state: 1 enabled, 0 disabled, -1 unknown"; + String SWITCH_STATE_DESC = "Switch state: 1 enabled, 0 disabled"; String MASTER_ACTIVE_TIME_DESC = "Master Active Time"; String MASTER_START_TIME_DESC = "Master Start Time"; String MASTER_FINISHED_INITIALIZATION_TIME_DESC = diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java index 5ab9035385a1..600cfc714695 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java @@ -166,8 +166,9 @@ public interface MetricsMasterWrapper { long getOldWALsDirSize(); /** - * Returns switch states (1 enabled, 0 disabled, -1 unknown), or an empty map unless this Master - * is active and initialized. Implementations must not perform remote reads. + * Returns known switch states (1 enabled, 0 disabled), or an empty map unless this Master is + * active and initialized. Unavailable states are omitted. Implementations must not perform remote + * reads. */ Map getSwitchStates(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java index 417bfeec8171..1fcd5c0c6094 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java @@ -275,25 +275,27 @@ public Map getSwitchStates() { if (!master.isActiveMaster() || !master.isInitialized()) { return Collections.emptyMap(); } - int rpcThrottleEnabled = 0; - if (QuotaUtil.isQuotaEnabled(master.getConfiguration())) { - rpcThrottleEnabled = -1; - if (master.getMasterQuotaManager() != null) { - rpcThrottleEnabled = master.getRpcThrottleStateStore().get() ? 1 : 0; - } + Map states = new HashMap<>(); + states.put(MetricsMasterSource.BALANCER_ENABLED_NAME, master.isBalancerOn() ? 1 : 0); + states.put(MetricsMasterSource.NORMALIZER_ENABLED_NAME, master.isNormalizerOn() ? 1 : 0); + states.put(MetricsMasterSource.SPLIT_ENABLED_NAME, + master.isSplitOrMergeEnabled(MasterSwitchType.SPLIT) ? 1 : 0); + states.put(MetricsMasterSource.MERGE_ENABLED_NAME, + master.isSplitOrMergeEnabled(MasterSwitchType.MERGE) ? 1 : 0); + states.put(MetricsMasterSource.CLEANER_CHORE_ENABLED_NAME, + master.isCleanerChoreEnabled() ? 1 : 0); + states.put(MetricsMasterSource.CATALOG_JANITOR_ENABLED_NAME, + master.isCatalogJanitorEnabled() ? 1 : 0); + states.put(MetricsMasterSource.SNAPSHOT_CLEANUP_ENABLED_NAME, + master.snapshotCleanupStateStore.get() ? 1 : 0); + states.put(MetricsMasterSource.REPLICATION_PEER_MODIFICATION_ENABLED_NAME, + master.isReplicationPeerModificationEnabled() ? 1 : 0); + if (!QuotaUtil.isQuotaEnabled(master.getConfiguration())) { + states.put(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, 0); + } else if (master.getMasterQuotaManager() != null) { + states.put(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, + master.getRpcThrottleStateStore().get() ? 1 : 0); } - return Map.of(MetricsMasterSource.BALANCER_ENABLED_NAME, master.isBalancerOn() ? 1 : 0, - MetricsMasterSource.NORMALIZER_ENABLED_NAME, master.isNormalizerOn() ? 1 : 0, - MetricsMasterSource.SPLIT_ENABLED_NAME, - master.isSplitOrMergeEnabled(MasterSwitchType.SPLIT) ? 1 : 0, - MetricsMasterSource.MERGE_ENABLED_NAME, - master.isSplitOrMergeEnabled(MasterSwitchType.MERGE) ? 1 : 0, - MetricsMasterSource.CLEANER_CHORE_ENABLED_NAME, master.isCleanerChoreEnabled() ? 1 : 0, - MetricsMasterSource.CATALOG_JANITOR_ENABLED_NAME, master.isCatalogJanitorEnabled() ? 1 : 0, - MetricsMasterSource.SNAPSHOT_CLEANUP_ENABLED_NAME, - master.snapshotCleanupStateStore.get() ? 1 : 0, - MetricsMasterSource.REPLICATION_PEER_MODIFICATION_ENABLED_NAME, - master.isReplicationPeerModificationEnabled() ? 1 : 0, - MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, rpcThrottleEnabled); + return states; } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterRepairMode.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterRepairMode.java index e37621c5b28f..5423ae2dca8c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterRepairMode.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterRepairMode.java @@ -17,15 +17,20 @@ */ package org.apache.hadoop.hbase.master; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Arrays; +import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import java.util.stream.StreamSupport; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CompatibilityFactory; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.StartTestingClusterOption; @@ -37,6 +42,8 @@ import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.quotas.QuotaUtil; +import org.apache.hadoop.hbase.test.MetricsAssertHelper; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.util.Bytes; @@ -96,6 +103,33 @@ public void testNewCluster() throws Exception { } } + @Test + public void testSwitchMetricsWithQuotasEnabled() throws Exception { + enableMaintenanceMode(); + TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true); + TEST_UTIL.startMiniCluster( + StartTestingClusterOption.builder().numRegionServers(0).numDataNodes(1).build()); + + HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster(); + assertTrue(master.isInMaintenanceMode()); + assertNull(master.getMasterQuotaManager()); + MetricsMasterSource source = master.getMasterMetrics().getMetricsSource(); + MetricsAssertHelper metricsHelper = CompatibilityFactory.getInstance(MetricsAssertHelper.class); + Map states = new MetricsMasterWrapperImpl(master).getSwitchStates(); + assertEquals(8, states.size()); + assertFalse(states.containsKey(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME)); + assertFalse( + metricsHelper.checkGaugeExists(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, source)); + states.forEach((name, state) -> { + assertTrue(state == 0 || state == 1); + metricsHelper.assertGauge(name, state.longValue(), source); + }); + metricsHelper.assertGauge(MetricsMasterSource.BALANCER_ENABLED_NAME, 0L, source); + metricsHelper.assertGauge(MetricsMasterSource.NORMALIZER_ENABLED_NAME, 0L, source); + metricsHelper.assertGauge(MetricsMasterSource.SPLIT_ENABLED_NAME, 0L, source); + metricsHelper.assertGauge(MetricsMasterSource.MERGE_ENABLED_NAME, 0L, source); + } + @Test public void testExistingCluster() throws Exception { TableName testRepairMode = TableName.valueOf(testMethodName); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterSwitchMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterSwitchMetrics.java index 44a5761374d9..032340e56c9c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterSwitchMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterSwitchMetrics.java @@ -128,12 +128,30 @@ public void testOnlyActiveInitializedMasterExposesSwitches() { } @Test - public void testQuotaDisabledAndUnknownStates() { + public void testRpcThrottleOmittedUntilQuotaManagerReady() { + MasterQuotaManager quotaManager = master.getMasterQuotaManager(); + Map expected = new HashMap<>(wrapper.getSwitchStates()); + expected.remove(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME); when(master.getMasterQuotaManager()).thenReturn(null); - assertEquals(-1, wrapper.getSwitchStates().get(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME)); + assertEquals(expected, wrapper.getSwitchStates()); + assertFalse( + METRICS_HELPER.checkGaugeExists(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, source)); + expected.forEach((name, state) -> METRICS_HELPER.assertGauge(name, state.longValue(), source)); + when(master.getMasterQuotaManager()).thenReturn(quotaManager); + when(rpcThrottleStateStore.get()).thenReturn(true); + expected.put(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, 1); + assertEquals(expected, wrapper.getSwitchStates()); + METRICS_HELPER.assertGauge(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, 1L, source); + } + + @Test + public void testRpcThrottleDisabledWithoutQuotaSupport() { conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, false); + when(master.getMasterQuotaManager()).thenReturn(null); when(rpcThrottleStateStore.get()).thenReturn(true); assertEquals(0, wrapper.getSwitchStates().get(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME)); + METRICS_HELPER.assertGauge(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, 0L, source); + verifyNoInteractions(rpcThrottleStateStore); } } diff --git a/hbase-website/app/pages/_docs/docs/_mdx/(multi-page)/operational-management/metrics-and-monitoring.mdx b/hbase-website/app/pages/_docs/docs/_mdx/(multi-page)/operational-management/metrics-and-monitoring.mdx index 55f5bf00417a..c3a59acf207e 100644 --- a/hbase-website/app/pages/_docs/docs/_mdx/(multi-page)/operational-management/metrics-and-monitoring.mdx +++ b/hbase-website/app/pages/_docs/docs/_mdx/(multi-page)/operational-management/metrics-and-monitoring.mdx @@ -137,8 +137,8 @@ The age of the longest region in transition, in milliseconds The active, initialized Master exposes numeric switch-state gauges under `Hadoop:service=HBase,name=Master,sub=Server`. A value of `1` means enabled and -`0` means disabled. A value of `-1` means the state is unknown. The gauges are -absent on backup Masters and before the Master is initialized. +`0` means disabled. The gauges are absent on backup Masters and before the Master +is initialized. Enabled states indicate that an operation is permitted, rather than that it is currently running. @@ -162,7 +162,8 @@ cleaner and the WAL cleaner. The existing `tag.isBalancerActive` tag under All switch states are read from memory during metrics collection. When quota support is disabled, `rpcThrottleEnabled` is `0`. With quota support -enabled, it is `-1` until the quota manager is ready. +enabled, it is omitted while the quota manager is unavailable, including in +Master maintenance mode. ### Most Important RegionServer Metrics From b0d816b6f09fee3a66ac468d0b8b499eeeabafc7 Mon Sep 17 00:00:00 2001 From: Junegunn Choi Date: Wed, 14 Oct 2026 00:33:23 +0900 Subject: [PATCH 5/6] HBASE-30473 Use metric constants in TestMasterMetrics --- .../hbase/master/TestMasterMetrics.java | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java index 1eaae5113d6a..cff8014eb24e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java @@ -154,14 +154,19 @@ public void testSwitchMetricsFollowAdminChanges() throws IOException { Admin admin = TEST_UTIL.getAdmin(); MetricsMasterSource source = master.getMasterMetrics().getMetricsSource(); Map> switches = new HashMap<>(); - switches.put("balancerEnabled", enabled -> admin.balancerSwitch(enabled, false)); - switches.put("normalizerEnabled", admin::normalizerSwitch); - switches.put("splitEnabled", enabled -> admin.splitSwitch(enabled, false)); - switches.put("mergeEnabled", enabled -> admin.mergeSwitch(enabled, false)); - switches.put("cleanerChoreEnabled", admin::cleanerChoreSwitch); - switches.put("catalogJanitorEnabled", admin::catalogJanitorSwitch); - switches.put("snapshotCleanupEnabled", enabled -> admin.snapshotCleanupSwitch(enabled, false)); - switches.put("replicationPeerModificationEnabled", admin::replicationPeerModificationSwitch); + switches.put(MetricsMasterSource.BALANCER_ENABLED_NAME, + enabled -> admin.balancerSwitch(enabled, false)); + switches.put(MetricsMasterSource.NORMALIZER_ENABLED_NAME, admin::normalizerSwitch); + switches.put(MetricsMasterSource.SPLIT_ENABLED_NAME, + enabled -> admin.splitSwitch(enabled, false)); + switches.put(MetricsMasterSource.MERGE_ENABLED_NAME, + enabled -> admin.mergeSwitch(enabled, false)); + switches.put(MetricsMasterSource.CLEANER_CHORE_ENABLED_NAME, admin::cleanerChoreSwitch); + switches.put(MetricsMasterSource.CATALOG_JANITOR_ENABLED_NAME, admin::catalogJanitorSwitch); + switches.put(MetricsMasterSource.SNAPSHOT_CLEANUP_ENABLED_NAME, + enabled -> admin.snapshotCleanupSwitch(enabled, false)); + switches.put(MetricsMasterSource.REPLICATION_PEER_MODIFICATION_ENABLED_NAME, + admin::replicationPeerModificationSwitch); Map initialStates = new MetricsMasterWrapperImpl(master).getSwitchStates(); for (Map.Entry> entry : switches.entrySet()) { try { @@ -174,7 +179,7 @@ public void testSwitchMetricsFollowAdminChanges() throws IOException { } } // Quota support is disabled by default. - metricsHelper.assertGauge("rpcThrottleEnabled", 0L, source); + metricsHelper.assertGauge(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, 0L, source); } @Test From c1e6eb47c26cca1929995b2e7d694fa6aefb9de5 Mon Sep 17 00:00:00 2001 From: Junegunn Choi Date: Thu, 15 Oct 2026 09:52:16 +0900 Subject: [PATCH 6/6] HBASE-30473 Preserve RPC throttle metric across config reloads --- .../master/MetricsMasterWrapperImpl.java | 10 ++-- .../hbase/quotas/MasterQuotaManager.java | 2 +- .../hbase/master/TestMasterSwitchMetrics.java | 51 +++++++++++++++++-- 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java index 1fcd5c0c6094..0076b3ff092f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.quotas.MasterQuotaManager; import org.apache.hadoop.hbase.quotas.QuotaObserverChore; import org.apache.hadoop.hbase.quotas.QuotaUtil; import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot; @@ -290,11 +291,12 @@ public Map getSwitchStates() { master.snapshotCleanupStateStore.get() ? 1 : 0); states.put(MetricsMasterSource.REPLICATION_PEER_MODIFICATION_ENABLED_NAME, master.isReplicationPeerModificationEnabled() ? 1 : 0); - if (!QuotaUtil.isQuotaEnabled(master.getConfiguration())) { - states.put(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, 0); - } else if (master.getMasterQuotaManager() != null) { + MasterQuotaManager quotaManager = master.getMasterQuotaManager(); + if (quotaManager != null) { states.put(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, - master.getRpcThrottleStateStore().get() ? 1 : 0); + quotaManager.isRpcThrottleEnabled() ? 1 : 0); + } else if (!QuotaUtil.isQuotaEnabled(master.getConfiguration())) { + states.put(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, 0); } return states; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java index 3f4ba9f59ed8..9ebce134777a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java @@ -427,7 +427,7 @@ public IsRpcThrottleEnabledResponse isRpcThrottleEnabled(IsRpcThrottleEnabledReq } } - public boolean isRpcThrottleEnabled() throws IOException { + public boolean isRpcThrottleEnabled() { return initialized && masterServices.getRpcThrottleStateStore().get(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterSwitchMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterSwitchMetrics.java index 032340e56c9c..8dc87e82a142 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterSwitchMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterSwitchMetrics.java @@ -24,12 +24,17 @@ import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.function.Consumer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.CompatibilityFactory; +import org.apache.hadoop.hbase.TableDescriptors; import org.apache.hadoop.hbase.client.MasterSwitchType; import org.apache.hadoop.hbase.master.snapshot.SnapshotCleanupStateStore; import org.apache.hadoop.hbase.metrics.MetricRegistries; @@ -44,6 +49,9 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; @Tag(MasterTests.TAG) @Tag(SmallTests.TAG) @@ -58,7 +66,7 @@ public class TestMasterSwitchMetrics { private MetricsMasterSourceImpl source; @BeforeEach - public void setUp() { + public void setUp() throws IOException { master = mock(HMaster.class); conf = new Configuration(false); conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true); @@ -66,7 +74,13 @@ public void setUp() { when(master.isActiveMaster()).thenReturn(true); when(master.isInitialized()).thenReturn(true); when(master.getMasterWalManager()).thenReturn(mock(MasterWalManager.class)); - when(master.getMasterQuotaManager()).thenReturn(mock(MasterQuotaManager.class)); + TableDescriptors tableDescriptors = mock(TableDescriptors.class); + when(tableDescriptors.exists(QuotaUtil.QUOTA_TABLE_NAME)).thenReturn(true); + when(master.getTableDescriptors()).thenReturn(tableDescriptors); + when(master.getClusterSchema()).thenReturn(mock(ClusterSchema.class)); + MasterQuotaManager quotaManager = new MasterQuotaManager(master); + quotaManager.start(); + when(master.getMasterQuotaManager()).thenReturn(quotaManager); master.snapshotCleanupStateStore = mock(SnapshotCleanupStateStore.class); rpcThrottleStateStore = mock(RpcThrottleStateStore.class); when(master.getRpcThrottleStateStore()).thenReturn(rpcThrottleStateStore); @@ -112,7 +126,6 @@ public void testEachSwitchChangesItsGauge() { .forEach((name, state) -> METRICS_HELPER.assertGauge(name, state.longValue(), source)); } } - verifyNoInteractions(master.getMasterQuotaManager()); } @Test @@ -154,4 +167,36 @@ public void testRpcThrottleDisabledWithoutQuotaSupport() { METRICS_HELPER.assertGauge(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, 0L, source); verifyNoInteractions(rpcThrottleStateStore); } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void testRpcThrottleUnaffectedByConfigurationReload(boolean quotaEnabled, + @TempDir Path tempDir) throws IOException { + Path site = tempDir.resolve("hbase-site.xml"); + writeQuotaConfiguration(site, quotaEnabled); + conf = new Configuration(false); + conf.addResource(new org.apache.hadoop.fs.Path(site.toUri())); + when(master.getConfiguration()).thenReturn(conf); + MasterQuotaManager quotaManager = new MasterQuotaManager(master); + quotaManager.start(); + when(master.getMasterQuotaManager()).thenReturn(quotaManager); + when(rpcThrottleStateStore.get()).thenReturn(true); + long expected = quotaEnabled ? 1L : 0L; + assertEquals(quotaEnabled, quotaManager.isRpcThrottleEnabled()); + METRICS_HELPER.assertGauge(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, expected, source); + + writeQuotaConfiguration(site, !quotaEnabled); + conf.reloadConfiguration(); + assertEquals(!quotaEnabled, QuotaUtil.isQuotaEnabled(conf)); + assertEquals(quotaEnabled, quotaManager.isRpcThrottleEnabled()); + METRICS_HELPER.assertGauge(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, expected, source); + } + + private static void writeQuotaConfiguration(Path path, boolean quotaEnabled) throws IOException { + Configuration config = new Configuration(false); + config.setBoolean(QuotaUtil.QUOTA_CONF_KEY, quotaEnabled); + try (OutputStream out = Files.newOutputStream(path)) { + config.writeXml(out); + } + } }