diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java index d606ed630881..cf47b8843f1b 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java @@ -71,6 +71,16 @@ public interface MetricsMasterSource extends BaseSource { String CLUSTER_READ_REQUESTS_NAME = "clusterReadRequests"; String CLUSTER_WRITE_REQUESTS_NAME = "clusterWriteRequests"; String OLD_WAL_DIR_SIZE_NAME = "oldWALsDirSize"; + String BALANCER_ENABLED_NAME = "balancerEnabled"; + String NORMALIZER_ENABLED_NAME = "normalizerEnabled"; + String SPLIT_ENABLED_NAME = "splitEnabled"; + String MERGE_ENABLED_NAME = "mergeEnabled"; + String CLEANER_CHORE_ENABLED_NAME = "cleanerChoreEnabled"; + String CATALOG_JANITOR_ENABLED_NAME = "catalogJanitorEnabled"; + String SNAPSHOT_CLEANUP_ENABLED_NAME = "snapshotCleanupEnabled"; + String REPLICATION_PEER_MODIFICATION_ENABLED_NAME = "replicationPeerModificationEnabled"; + String RPC_THROTTLE_ENABLED_NAME = "rpcThrottleEnabled"; + String SWITCH_STATE_DESC = "Switch state: 1 enabled, 0 disabled"; String MASTER_ACTIVE_TIME_DESC = "Master Active Time"; String MASTER_START_TIME_DESC = "Master Start Time"; String MASTER_FINISHED_INITIALIZATION_TIME_DESC = diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java index 011e66312aa3..d2542efbaecd 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java @@ -132,6 +132,9 @@ public void getMetrics(MetricsCollector metricsCollector, boolean all) { String.valueOf(masterWrapper.getIsActiveMaster())) .addGauge(Interns.info(OLD_WAL_DIR_SIZE_NAME, OLD_WAL_DIR_SIZE_DESC), masterWrapper.getOldWALsDirSize()); + + masterWrapper.getSwitchStates().forEach((name, state) -> metricsRecordBuilder + .addGauge(Interns.info(name, SWITCH_STATE_DESC), state)); } metricsRegistry.snapshot(metricsRecordBuilder, all); diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java index ad8728327014..600cfc714695 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java @@ -164,4 +164,11 @@ public interface MetricsMasterWrapper { * Get the size of old WALs directory in bytes. */ long getOldWALsDirSize(); + + /** + * Returns known switch states (1 enabled, 0 disabled), or an empty map unless this Master is + * active and initialized. Unavailable states are omitted. Implementations must not perform remote + * reads. + */ + Map getSwitchStates(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java index 67cbbdb8ae9e..0076b3ff092f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java @@ -27,12 +27,15 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.MasterSwitchType; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.quotas.MasterQuotaManager; import org.apache.hadoop.hbase.quotas.QuotaObserverChore; +import org.apache.hadoop.hbase.quotas.QuotaUtil; import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.PairOfSameType; @@ -267,4 +270,34 @@ public long getOldWALsDirSize() { } return master.getMasterWalManager().getOldWALsDirSize(); } + + @Override + public Map getSwitchStates() { + if (!master.isActiveMaster() || !master.isInitialized()) { + return Collections.emptyMap(); + } + Map states = new HashMap<>(); + states.put(MetricsMasterSource.BALANCER_ENABLED_NAME, master.isBalancerOn() ? 1 : 0); + states.put(MetricsMasterSource.NORMALIZER_ENABLED_NAME, master.isNormalizerOn() ? 1 : 0); + states.put(MetricsMasterSource.SPLIT_ENABLED_NAME, + master.isSplitOrMergeEnabled(MasterSwitchType.SPLIT) ? 1 : 0); + states.put(MetricsMasterSource.MERGE_ENABLED_NAME, + master.isSplitOrMergeEnabled(MasterSwitchType.MERGE) ? 1 : 0); + states.put(MetricsMasterSource.CLEANER_CHORE_ENABLED_NAME, + master.isCleanerChoreEnabled() ? 1 : 0); + states.put(MetricsMasterSource.CATALOG_JANITOR_ENABLED_NAME, + master.isCatalogJanitorEnabled() ? 1 : 0); + states.put(MetricsMasterSource.SNAPSHOT_CLEANUP_ENABLED_NAME, + master.snapshotCleanupStateStore.get() ? 1 : 0); + states.put(MetricsMasterSource.REPLICATION_PEER_MODIFICATION_ENABLED_NAME, + master.isReplicationPeerModificationEnabled() ? 1 : 0); + MasterQuotaManager quotaManager = master.getMasterQuotaManager(); + if (quotaManager != null) { + states.put(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, + quotaManager.isRpcThrottleEnabled() ? 1 : 0); + } else if (!QuotaUtil.isQuotaEnabled(master.getConfiguration())) { + states.put(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, 0); + } + return states; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java index 3f4ba9f59ed8..9ebce134777a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java @@ -427,7 +427,7 @@ public IsRpcThrottleEnabledResponse isRpcThrottleEnabled(IsRpcThrottleEnabledReq } } - public boolean isRpcThrottleEnabled() throws IOException { + public boolean isRpcThrottleEnabled() { return initialized && masterServices.getRpcThrottleStateStore().get(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java index 46cc88825bc3..cff8014eb24e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.StartTestingClusterOption; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.YouAreDeadException; +import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.RegionStatesCount; import org.apache.hadoop.hbase.client.TableDescriptor; @@ -51,6 +52,7 @@ import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.FSTableDescriptors; +import org.apache.hadoop.hbase.util.IOExceptionConsumer; import org.apache.zookeeper.KeeperException; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -147,6 +149,39 @@ public static void after() throws Exception { TEST_UTIL.shutdownMiniCluster(); } + @Test + public void testSwitchMetricsFollowAdminChanges() throws IOException { + Admin admin = TEST_UTIL.getAdmin(); + MetricsMasterSource source = master.getMasterMetrics().getMetricsSource(); + Map> switches = new HashMap<>(); + switches.put(MetricsMasterSource.BALANCER_ENABLED_NAME, + enabled -> admin.balancerSwitch(enabled, false)); + switches.put(MetricsMasterSource.NORMALIZER_ENABLED_NAME, admin::normalizerSwitch); + switches.put(MetricsMasterSource.SPLIT_ENABLED_NAME, + enabled -> admin.splitSwitch(enabled, false)); + switches.put(MetricsMasterSource.MERGE_ENABLED_NAME, + enabled -> admin.mergeSwitch(enabled, false)); + switches.put(MetricsMasterSource.CLEANER_CHORE_ENABLED_NAME, admin::cleanerChoreSwitch); + switches.put(MetricsMasterSource.CATALOG_JANITOR_ENABLED_NAME, admin::catalogJanitorSwitch); + switches.put(MetricsMasterSource.SNAPSHOT_CLEANUP_ENABLED_NAME, + enabled -> admin.snapshotCleanupSwitch(enabled, false)); + switches.put(MetricsMasterSource.REPLICATION_PEER_MODIFICATION_ENABLED_NAME, + admin::replicationPeerModificationSwitch); + Map initialStates = new MetricsMasterWrapperImpl(master).getSwitchStates(); + for (Map.Entry> entry : switches.entrySet()) { + try { + for (boolean enabled : new boolean[] { false, true }) { + entry.getValue().accept(enabled); + metricsHelper.assertGauge(entry.getKey(), enabled ? 1L : 0L, source); + } + } finally { + entry.getValue().accept(initialStates.get(entry.getKey()) == 1); + } + } + // Quota support is disabled by default. + metricsHelper.assertGauge(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, 0L, source); + } + @Test public void testClusterRequests() throws Exception { // sending fake request to master to see how metric value has changed diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterRepairMode.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterRepairMode.java index e37621c5b28f..5423ae2dca8c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterRepairMode.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterRepairMode.java @@ -17,15 +17,20 @@ */ package org.apache.hadoop.hbase.master; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Arrays; +import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import java.util.stream.StreamSupport; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CompatibilityFactory; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.StartTestingClusterOption; @@ -37,6 +42,8 @@ import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.quotas.QuotaUtil; +import org.apache.hadoop.hbase.test.MetricsAssertHelper; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.util.Bytes; @@ -96,6 +103,33 @@ public void testNewCluster() throws Exception { } } + @Test + public void testSwitchMetricsWithQuotasEnabled() throws Exception { + enableMaintenanceMode(); + TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true); + TEST_UTIL.startMiniCluster( + StartTestingClusterOption.builder().numRegionServers(0).numDataNodes(1).build()); + + HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster(); + assertTrue(master.isInMaintenanceMode()); + assertNull(master.getMasterQuotaManager()); + MetricsMasterSource source = master.getMasterMetrics().getMetricsSource(); + MetricsAssertHelper metricsHelper = CompatibilityFactory.getInstance(MetricsAssertHelper.class); + Map states = new MetricsMasterWrapperImpl(master).getSwitchStates(); + assertEquals(8, states.size()); + assertFalse(states.containsKey(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME)); + assertFalse( + metricsHelper.checkGaugeExists(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, source)); + states.forEach((name, state) -> { + assertTrue(state == 0 || state == 1); + metricsHelper.assertGauge(name, state.longValue(), source); + }); + metricsHelper.assertGauge(MetricsMasterSource.BALANCER_ENABLED_NAME, 0L, source); + metricsHelper.assertGauge(MetricsMasterSource.NORMALIZER_ENABLED_NAME, 0L, source); + metricsHelper.assertGauge(MetricsMasterSource.SPLIT_ENABLED_NAME, 0L, source); + metricsHelper.assertGauge(MetricsMasterSource.MERGE_ENABLED_NAME, 0L, source); + } + @Test public void testExistingCluster() throws Exception { TableName testRepairMode = TableName.valueOf(testMethodName); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterSwitchMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterSwitchMetrics.java new file mode 100644 index 000000000000..8dc87e82a142 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterSwitchMetrics.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CompatibilityFactory; +import org.apache.hadoop.hbase.TableDescriptors; +import org.apache.hadoop.hbase.client.MasterSwitchType; +import org.apache.hadoop.hbase.master.snapshot.SnapshotCleanupStateStore; +import org.apache.hadoop.hbase.metrics.MetricRegistries; +import org.apache.hadoop.hbase.quotas.MasterQuotaManager; +import org.apache.hadoop.hbase.quotas.QuotaUtil; +import org.apache.hadoop.hbase.quotas.RpcThrottleStateStore; +import org.apache.hadoop.hbase.test.MetricsAssertHelper; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) +public class TestMasterSwitchMetrics { + private static final MetricsAssertHelper METRICS_HELPER = + CompatibilityFactory.getInstance(MetricsAssertHelper.class); + + private HMaster master; + private Configuration conf; + private RpcThrottleStateStore rpcThrottleStateStore; + private MetricsMasterWrapperImpl wrapper; + private MetricsMasterSourceImpl source; + + @BeforeEach + public void setUp() throws IOException { + master = mock(HMaster.class); + conf = new Configuration(false); + conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true); + when(master.getConfiguration()).thenReturn(conf); + when(master.isActiveMaster()).thenReturn(true); + when(master.isInitialized()).thenReturn(true); + when(master.getMasterWalManager()).thenReturn(mock(MasterWalManager.class)); + TableDescriptors tableDescriptors = mock(TableDescriptors.class); + when(tableDescriptors.exists(QuotaUtil.QUOTA_TABLE_NAME)).thenReturn(true); + when(master.getTableDescriptors()).thenReturn(tableDescriptors); + when(master.getClusterSchema()).thenReturn(mock(ClusterSchema.class)); + MasterQuotaManager quotaManager = new MasterQuotaManager(master); + quotaManager.start(); + when(master.getMasterQuotaManager()).thenReturn(quotaManager); + master.snapshotCleanupStateStore = mock(SnapshotCleanupStateStore.class); + rpcThrottleStateStore = mock(RpcThrottleStateStore.class); + when(master.getRpcThrottleStateStore()).thenReturn(rpcThrottleStateStore); + wrapper = new MetricsMasterWrapperImpl(master); + source = new MetricsMasterSourceImpl(wrapper); + } + + @AfterEach + public void tearDown() { + DefaultMetricsSystem.instance().unregisterSource(MetricsMasterSource.METRICS_JMX_CONTEXT); + MetricRegistries.global().remove(source.getMetricRegistryInfo()); + } + + @Test + public void testEachSwitchChangesItsGauge() { + Map> switches = new HashMap<>(); + switches.put(MetricsMasterSource.BALANCER_ENABLED_NAME, + enabled -> when(master.isBalancerOn()).thenReturn(enabled)); + switches.put(MetricsMasterSource.NORMALIZER_ENABLED_NAME, + enabled -> when(master.isNormalizerOn()).thenReturn(enabled)); + switches.put(MetricsMasterSource.SPLIT_ENABLED_NAME, + enabled -> when(master.isSplitOrMergeEnabled(MasterSwitchType.SPLIT)).thenReturn(enabled)); + switches.put(MetricsMasterSource.MERGE_ENABLED_NAME, + enabled -> when(master.isSplitOrMergeEnabled(MasterSwitchType.MERGE)).thenReturn(enabled)); + switches.put(MetricsMasterSource.CLEANER_CHORE_ENABLED_NAME, + enabled -> when(master.isCleanerChoreEnabled()).thenReturn(enabled)); + switches.put(MetricsMasterSource.CATALOG_JANITOR_ENABLED_NAME, + enabled -> when(master.isCatalogJanitorEnabled()).thenReturn(enabled)); + switches.put(MetricsMasterSource.SNAPSHOT_CLEANUP_ENABLED_NAME, + enabled -> when(master.snapshotCleanupStateStore.get()).thenReturn(enabled)); + switches.put(MetricsMasterSource.REPLICATION_PEER_MODIFICATION_ENABLED_NAME, + enabled -> when(master.isReplicationPeerModificationEnabled()).thenReturn(enabled)); + switches.put(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, + enabled -> when(rpcThrottleStateStore.get()).thenReturn(enabled)); + Map expected = new HashMap<>(); + switches.keySet().forEach(name -> expected.put(name, 0)); + for (Map.Entry> entry : switches.entrySet()) { + for (boolean enabled : new boolean[] { true, false }) { + entry.getValue().accept(enabled); + expected.put(entry.getKey(), enabled ? 1 : 0); + assertEquals(expected, wrapper.getSwitchStates()); + expected + .forEach((name, state) -> METRICS_HELPER.assertGauge(name, state.longValue(), source)); + } + } + } + + @Test + public void testOnlyActiveInitializedMasterExposesSwitches() { + Set metricNames = wrapper.getSwitchStates().keySet(); + when(master.isInitialized()).thenReturn(false); + assertTrue(wrapper.getSwitchStates().isEmpty()); + metricNames.forEach(name -> assertFalse(METRICS_HELPER.checkGaugeExists(name, source))); + when(master.isInitialized()).thenReturn(true); + when(master.isActiveMaster()).thenReturn(false); + assertTrue(wrapper.getSwitchStates().isEmpty()); + metricNames.forEach(name -> assertFalse(METRICS_HELPER.checkGaugeExists(name, source))); + } + + @Test + public void testRpcThrottleOmittedUntilQuotaManagerReady() { + MasterQuotaManager quotaManager = master.getMasterQuotaManager(); + Map expected = new HashMap<>(wrapper.getSwitchStates()); + expected.remove(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME); + when(master.getMasterQuotaManager()).thenReturn(null); + assertEquals(expected, wrapper.getSwitchStates()); + assertFalse( + METRICS_HELPER.checkGaugeExists(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, source)); + expected.forEach((name, state) -> METRICS_HELPER.assertGauge(name, state.longValue(), source)); + + when(master.getMasterQuotaManager()).thenReturn(quotaManager); + when(rpcThrottleStateStore.get()).thenReturn(true); + expected.put(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, 1); + assertEquals(expected, wrapper.getSwitchStates()); + METRICS_HELPER.assertGauge(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, 1L, source); + } + + @Test + public void testRpcThrottleDisabledWithoutQuotaSupport() { + conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, false); + when(master.getMasterQuotaManager()).thenReturn(null); + when(rpcThrottleStateStore.get()).thenReturn(true); + assertEquals(0, wrapper.getSwitchStates().get(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME)); + METRICS_HELPER.assertGauge(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, 0L, source); + verifyNoInteractions(rpcThrottleStateStore); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void testRpcThrottleUnaffectedByConfigurationReload(boolean quotaEnabled, + @TempDir Path tempDir) throws IOException { + Path site = tempDir.resolve("hbase-site.xml"); + writeQuotaConfiguration(site, quotaEnabled); + conf = new Configuration(false); + conf.addResource(new org.apache.hadoop.fs.Path(site.toUri())); + when(master.getConfiguration()).thenReturn(conf); + MasterQuotaManager quotaManager = new MasterQuotaManager(master); + quotaManager.start(); + when(master.getMasterQuotaManager()).thenReturn(quotaManager); + when(rpcThrottleStateStore.get()).thenReturn(true); + long expected = quotaEnabled ? 1L : 0L; + assertEquals(quotaEnabled, quotaManager.isRpcThrottleEnabled()); + METRICS_HELPER.assertGauge(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, expected, source); + + writeQuotaConfiguration(site, !quotaEnabled); + conf.reloadConfiguration(); + assertEquals(!quotaEnabled, QuotaUtil.isQuotaEnabled(conf)); + assertEquals(quotaEnabled, quotaManager.isRpcThrottleEnabled()); + METRICS_HELPER.assertGauge(MetricsMasterSource.RPC_THROTTLE_ENABLED_NAME, expected, source); + } + + private static void writeQuotaConfiguration(Path path, boolean quotaEnabled) throws IOException { + Configuration config = new Configuration(false); + config.setBoolean(QuotaUtil.QUOTA_CONF_KEY, quotaEnabled); + try (OutputStream out = Files.newOutputStream(path)) { + config.writeXml(out); + } + } +} diff --git a/hbase-website/app/pages/_docs/docs/_mdx/(multi-page)/operational-management/metrics-and-monitoring.mdx b/hbase-website/app/pages/_docs/docs/_mdx/(multi-page)/operational-management/metrics-and-monitoring.mdx index 88f8957c91fe..c3a59acf207e 100644 --- a/hbase-website/app/pages/_docs/docs/_mdx/(multi-page)/operational-management/metrics-and-monitoring.mdx +++ b/hbase-website/app/pages/_docs/docs/_mdx/(multi-page)/operational-management/metrics-and-monitoring.mdx @@ -133,6 +133,38 @@ The number of regions that have been in transition longer than a threshold time **hbase.master.ritOldestAge** The age of the longest region in transition, in milliseconds +#### Master switch states + +The active, initialized Master exposes numeric switch-state gauges under +`Hadoop:service=HBase,name=Master,sub=Server`. A value of `1` means enabled and +`0` means disabled. The gauges are absent on backup Masters and before the Master +is initialized. +Enabled states indicate that an operation is permitted, rather than that it is +currently running. + +| Gauge | HBase shell command | +| ------------------------------------ | ---------------------------------------------- | +| `balancerEnabled` | `balance_switch true\|false` | +| `normalizerEnabled` | `normalizer_switch true\|false` | +| `splitEnabled` | `splitormerge_switch 'SPLIT', true\|false` | +| `mergeEnabled` | `splitormerge_switch 'MERGE', true\|false` | +| `cleanerChoreEnabled` | `cleaner_chore_switch true\|false` | +| `catalogJanitorEnabled` | `catalogjanitor_switch true\|false` | +| `snapshotCleanupEnabled` | `snapshot_cleanup_switch true\|false` | +| `replicationPeerModificationEnabled` | `peer_modification_switch true\|false` | +| `rpcThrottleEnabled` | `enable_rpc_throttle` / `disable_rpc_throttle` | + +The gauges follow the corresponding enabled-state queries. Maintenance mode +disables the balancer, normalizer, split, and merge states. Per-table settings can +further restrict these operations. The cleaner state covers the default HFile +cleaner and the WAL cleaner. The existing `tag.isBalancerActive` tag under +`Master,sub=Balancer` remains available. + +All switch states are read from memory during metrics collection. +When quota support is disabled, `rpcThrottleEnabled` is `0`. With quota support +enabled, it is omitted while the quota manager is unavailable, including in +Master maintenance mode. + ### Most Important RegionServer Metrics Note: Counts are usually over the last metrics reporting interval.