diff --git a/debian/control b/debian/control index cdf663ef8906..0520f3487796 100644 --- a/debian/control +++ b/debian/control @@ -24,7 +24,7 @@ Description: CloudStack server library Package: cloudstack-agent Architecture: all -Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, ovmf, swtpm, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd, sysstat, python3-libnbd, socat +Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, ovmf, swtpm, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd, sysstat, python3-libnbd, python3-rados, python3-rbd, socat Recommends: init-system-helpers Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts Description: CloudStack agent diff --git a/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java b/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java index 53bfcce27038..f4f8d342eda7 100644 --- a/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java +++ b/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java @@ -33,7 +33,8 @@ */ public interface HighAvailabilityManager extends Manager { - List LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT = List.of(StoragePoolType.NetworkFilesystem, StoragePoolType.SharedMountPoint); + List LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT = List.of(StoragePoolType.NetworkFilesystem, StoragePoolType.SharedMountPoint, + StoragePoolType.RBD); ConfigKey ForceHA = new ConfigKey<>("Advanced", Boolean.class, "force.ha", "false", "Force High-Availability to happen even if the VM says no.", true, Cluster); diff --git a/packaging/el8/cloud.spec b/packaging/el8/cloud.spec index 9d8cab846164..352455180a05 100644 --- a/packaging/el8/cloud.spec +++ b/packaging/el8/cloud.spec @@ -128,6 +128,8 @@ Requires: (libgcrypt > 1.8.3 or libgcrypt20) Requires: (selinux-tools if selinux-tools) Requires: sysstat Requires: python3-libnbd +Requires: python3-rados +Requires: python3-rbd Requires: socat Provides: cloud-agent Group: System Environment/Libraries diff --git a/packaging/suse15/cloud.spec b/packaging/suse15/cloud.spec index 04ab66904287..9a22f20b8164 100644 --- a/packaging/suse15/cloud.spec +++ b/packaging/suse15/cloud.spec @@ -120,6 +120,8 @@ Requires: cifs-utils Requires: (edk2-ovmf or qemu-ovmf-x86_64) Requires: swtpm Requires: (python3-libvirt or python3-libvirt-python) +Requires: (python3-rados or python3-ceph-common) +Requires: (python3-rbd or python3-ceph-common) Requires: (qemu-img or qemu-tools) Requires: qemu-kvm Requires: cryptsetup diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java index 0ee59c95da39..08b5851d83f6 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java @@ -53,7 +53,7 @@ public Boolean hasHeartBeat() { } if (!heartBeatCheckResult) { - logger.warn("All checks with KVMHAChecker for {} considered it as dead. It may cause a shutdown of the host.", hostAndPools); + logger.warn("All checks with KVMHAChecker for {} considered it as dead, which may trigger HA actions, such as fencing it and restarting its VMs on other hosts.", hostAndPools); } return heartBeatCheckResult; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java index 9f1b849e9727..46499a0c4c07 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java @@ -20,6 +20,7 @@ import com.cloud.agent.properties.AgentPropertiesFileHandler; import com.cloud.ha.HighAvailabilityManager; import com.cloud.utils.script.Script; +import org.apache.commons.lang3.StringUtils; import org.libvirt.Connect; import org.libvirt.LibvirtException; import org.libvirt.StoragePool; @@ -54,7 +55,11 @@ public void removeStoragePool(String uuid) { synchronized (haStoragePools) { HAStoragePool pool = haStoragePools.get(uuid); if (pool != null) { - Script.runSimpleBashScript("umount " + pool.getMountDestPath()); + if (StringUtils.isNotEmpty(pool.getMountDestPath())) { + Script umount = new Script("umount", logger); + umount.add(pool.getMountDestPath()); + umount.execute(); + } haStoragePools.remove(uuid); } } @@ -130,7 +135,7 @@ private void checkForNotExistingLibvirtStoragePools(Set removedPools, St removedPools.add(uuid); } - logger.debug("Found NFS storage pool [{}] in libvirt, continuing.", uuid); + logger.debug("Found storage pool [{}] in libvirt, continuing.", uuid); } catch (LibvirtException e) { logger.debug("Failed to lookup libvirt storage pool [{}].", uuid, e); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java index b132f6d98ce9..745686ce33de 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java @@ -22,6 +22,8 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.CheckVMActivityOnStoragePoolCommand; import com.cloud.agent.api.to.StorageFilerTO; +import com.cloud.agent.properties.AgentProperties; +import com.cloud.agent.properties.AgentPropertiesFileHandler; import com.cloud.hypervisor.kvm.resource.KVMHABase.HAStoragePool; import com.cloud.hypervisor.kvm.storage.KVMStoragePool; import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; @@ -30,6 +32,8 @@ import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; +import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.utils.script.Script; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -50,7 +54,11 @@ public Answer execute(final CheckVMActivityOnStoragePoolCommand command, final L if (primaryPool.isPoolSupportHA()) { final HAStoragePool haPool = monitor.getStoragePool(pool.getUuid()); - final KVMHAVMActivityChecker ha = new KVMHAVMActivityChecker(haPool, command.getHost(), command.getVolumeList(), libvirtComputingResource.getVmActivityCheckPath(), command.getSuspectTimeInSeconds()); + final String vmActivityCheckPath = getVmActivityCheckPath(pool.getType(), libvirtComputingResource); + if (vmActivityCheckPath == null) { + return new Answer(command, false, "CheckVMActivityOnStoragePoolCommand: unable to find the script which checks the VM activity on a storage pool of type " + pool.getType()); + } + final KVMHAVMActivityChecker ha = new KVMHAVMActivityChecker(haPool, command.getHost(), command.getVolumeList(), vmActivityCheckPath, command.getSuspectTimeInSeconds()); final Future future = executors.submit(ha); try { final Boolean result = future.get(); @@ -67,4 +75,16 @@ public Answer execute(final CheckVMActivityOnStoragePoolCommand command, final L } return new Answer(command, false, "Unsupported Storage"); } + + /** + * The script which checks the VM activity depends on the type of the storage pool: a RBD pool has no shared mount point to look at, + * so its script checks the watchers of the volumes, instead of the files of the volumes as the script of the other pools. + */ + private String getVmActivityCheckPath(final StoragePoolType poolType, final LibvirtComputingResource libvirtComputingResource) { + if (StoragePoolType.RBD.equals(poolType)) { + final String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR); + return Script.findScript(kvmScriptsDir, "kvmvmactivity_rbd.py"); + } + return libvirtComputingResource.getVmActivityCheckPath(); + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java index d99847fd921a..c56171577e09 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java @@ -37,6 +37,7 @@ import com.cloud.agent.api.to.DiskTO; import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.ha.HighAvailabilityManager; import com.cloud.hypervisor.kvm.resource.KVMHABase; import com.cloud.hypervisor.kvm.resource.KVMHABase.PoolType; import com.cloud.hypervisor.kvm.resource.KVMHAMonitor; @@ -445,7 +446,7 @@ public boolean disconnectPhysicalDisk(StoragePoolType type, String poolUuid, Str public synchronized boolean deleteStoragePool(StoragePoolType type, String uuid) { StorageAdaptor adaptor = getStorageAdaptor(type); - if (type == StoragePoolType.NetworkFilesystem) { + if (HighAvailabilityManager.LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT.contains(type)) { _haMonitor.removeStoragePool(uuid); } boolean deleteStatus = adaptor.deleteStoragePool(uuid);; @@ -457,7 +458,7 @@ public synchronized boolean deleteStoragePool(StoragePoolType type, String uuid) public boolean deleteStoragePool(StoragePoolType type, String uuid, Map details) { StorageAdaptor adaptor = getStorageAdaptor(type); - if (type == StoragePoolType.NetworkFilesystem) { + if (HighAvailabilityManager.LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT.contains(type)) { _haMonitor.removeStoragePool(uuid); } boolean deleteStatus = adaptor.deleteStoragePool(uuid, details); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java index a8c32baa6ef3..12caa7e55abf 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java @@ -17,10 +17,15 @@ package com.cloud.hypervisor.kvm.storage; import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; import java.util.List; import java.util.Map; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.joda.time.Duration; @@ -40,6 +45,9 @@ import com.cloud.utils.script.Script; public class LibvirtStoragePool implements KVMStoragePool { + /** The environment variable which gives the Ceph key to the RBD heartbeat and VM-activity scripts. */ + protected static final String RBD_KEY_ENVIRONMENT_VARIABLE = "CEPH_KEY"; + protected Logger logger = LogManager.getLogger(getClass()); protected String uuid; protected long capacity; @@ -330,29 +338,130 @@ public boolean isPoolSupportHA() { public String getHearthBeatPath() { if (StoragePoolType.NetworkFilesystem.equals(type)) { - String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR); - String scriptPath = Script.findScript(kvmScriptsDir, "kvmheartbeat.sh"); - if (scriptPath == null) { - throw new CloudRuntimeException("Unable to find heartbeat script 'kvmheartbeat.sh' in directory: " + kvmScriptsDir); - } - return scriptPath; + return findKvmHaScript("kvmheartbeat.sh"); } else if (StoragePoolType.SharedMountPoint.equals(type)) { - String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR); - String scriptPath = Script.findScript(kvmScriptsDir, "kvmsmpheartbeat.sh"); - if (scriptPath == null) { - throw new CloudRuntimeException("Unable to find heartbeat script 'kvmsmpheartbeat.sh' in directory: " + kvmScriptsDir); + return findKvmHaScript("kvmsmpheartbeat.sh"); + } else if (StoragePoolType.RBD.equals(type)) { + return findKvmHaScript("kvmheartbeat_rbd.py"); + } + return null; + } + + private String findKvmHaScript(String scriptName) { + String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR); + String scriptPath = Script.findScript(kvmScriptsDir, scriptName); + if (scriptPath == null) { + throw new CloudRuntimeException(String.format("Unable to find script '%s' in directory: %s", scriptName, kvmScriptsDir)); + } + return scriptPath; + } + + /** + * Returns the Ceph monitors as expected by "--mon-host": the comma-separated monitors of the pool, trimmed + * and without empty entries. If the pool has a monitor port, it is added to each monitor which has none yet. + */ + protected String getRbdMonitors() { + List monitors = new ArrayList<>(); + for (String monitor : sourceHost.split(",")) { + monitor = monitor.trim(); + if (monitor.isEmpty()) { + continue; } - return scriptPath; + monitors.add(sourcePort > 0 ? addPortToRbdMonitor(monitor) : monitor); + } + return String.join(",", monitors); + } + + private String addPortToRbdMonitor(String monitor) { + if (monitor.startsWith("[")) { + // IPv6 address in square brackets, which has a port if followed by ":" + return monitor.contains("]:") ? monitor : monitor + ":" + sourcePort; + } + int colons = StringUtils.countMatches(monitor, ":"); + if (colons == 0) { + return monitor + ":" + sourcePort; + } + if (colons == 1) { + // IPv4 address or host name, with a port + return monitor; + } + // IPv6 address without square brackets, so without a port + return "[" + monitor + "]:" + sourcePort; + } + + /** + * Returns the environment of a heartbeat/VM-activity check script: for a RBD storage pool with cephx, the + * Ceph key, which is not an argument of the script as it would be visible in the process list. + */ + protected Map getHeartBeatScriptEnvironment() { + if (StoragePoolType.RBD.equals(type) && StringUtils.isNotBlank(authSecret)) { + return Map.of(RBD_KEY_ENVIRONMENT_VARIABLE, authSecret); } return null; } + /** + * Adds the Ceph cluster connection details (monitors, pool and, if cephx is enabled, the user; the key is in + * {@link #getHeartBeatScriptEnvironment()}) to a heartbeat/VM-activity check {@link Script} for a RBD storage pool. Mirrors the "mon_host"/"id"/"key" + * options that qemu itself uses to talk to RBD (see {@link KVMPhysicalDisk#RBDStringBuilder}). + */ + protected void addRbdConnectionArgs(Script cmd) { + boolean hasUser = StringUtils.isNotBlank(authUsername); + boolean hasSecret = StringUtils.isNotBlank(authSecret); + if (hasUser != hasSecret) { + throw new CloudRuntimeException(String.format("The Ceph user and key must be either both set or both unset for the storage pool %s", uuid)); + } + cmd.add("-s", getRbdMonitors()); + cmd.add("-o", sourceDir); + if (hasUser) { + cmd.add("-n", authUsername); + } + } + + /** + * Adds the arguments identifying the storage to a heartbeat/VM-activity check {@link Script}: + * the Ceph connection details for a RBD pool, or the NFS server, path and mount point otherwise. + */ + private void addPoolConnectionArgs(Script cmd, HAStoragePool pool) { + if (StoragePoolType.RBD.equals(type)) { + addRbdConnectionArgs(cmd); + } else { + cmd.add("-i", pool.getPoolIp()); + cmd.add("-p", pool.getPoolMountSourcePath()); + cmd.add("-m", pool.getMountDestPath()); + } + } + + /** + * Adds the volumes of the host to a VM-activity check {@link Script}. For a RBD storage pool they are given in + * a file, as they can be too many for an argument of a command (the limit is 128 KiB, about 3500 volumes), + * which would also be written to the log, with all the volumes, at each check. Otherwise, or if the file can not + * be written, they are an argument. + * + * @return the file, which has to be deleted after the script ran, or null if there is none + */ + protected File addVolumeListArgs(Script cmd, String volumes) { + if (StoragePoolType.RBD.equals(type)) { + File volumeListFile = null; + try { + volumeListFile = File.createTempFile("kvmha-volumes-", ".txt"); + Files.write(volumeListFile.toPath(), volumes.getBytes(StandardCharsets.UTF_8)); + cmd.add("-U", volumeListFile.getAbsolutePath()); + return volumeListFile; + } catch (IOException e) { + logger.warn("Unable to write the list of volumes to a file, giving it to the script as an argument: {}", e.getMessage()); + if (volumeListFile != null && !volumeListFile.delete()) { + logger.debug("Unable to delete the file {}", volumeListFile); + } + } + } + cmd.add("-u", volumes); + return null; + } public String createHeartBeatCommand(HAStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation) { Script cmd = new Script(primaryStoragePool.getPool().getHearthBeatPath(), HeartBeatUpdateTimeoutInMs, logger); - cmd.add("-i", primaryStoragePool.getPoolIp()); - cmd.add("-p", primaryStoragePool.getPoolMountSourcePath()); - cmd.add("-m", primaryStoragePool.getMountDestPath()); + addPoolConnectionArgs(cmd, primaryStoragePool); if (hostValidation) { cmd.add("-h", hostPrivateIp); @@ -360,7 +469,7 @@ public String createHeartBeatCommand(HAStoragePool primaryStoragePool, String ho cmd.add("-c"); } - return cmd.execute(); + return cmd.execute(new OutputInterpreter.OutputLogger(logger), getHeartBeatScriptEnvironment()); } @Override @@ -377,20 +486,18 @@ public String getStorageNodeId() { public Boolean hasHeartBeat(HAStoragePool pool, HostTO host) { String hostIp = host.getPrivateNetwork().getIp(); Script cmd = new Script(getHearthBeatPath(), HeartBeatCheckerTimeoutInMs, logger); - cmd.add("-i", pool.getPoolIp()); - cmd.add("-p", pool.getPoolMountSourcePath()); - cmd.add("-m", pool.getMountDestPath()); + addPoolConnectionArgs(cmd, pool); cmd.add("-h", hostIp); cmd.add("-r"); cmd.add("-t", String.valueOf(HeartBeatUpdateFreqInMs / 1000)); OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); - String result = cmd.execute(parser); + String result = cmd.execute(parser, getHeartBeatScriptEnvironment()); String parsedLine = parser.getLine(); logger.debug("Checking heart beat for host IP {} with KVMHAChecker [{command=\"{}\", result: \"{}\", log: \"{}\", pool: \"{}\"}].", hostIp, cmd.toString(), result, parsedLine, pool.getPoolIp()); if (result == null && parsedLine.contains("DEAD")) { - logger.warn("Checking heart beat for host IP {} with KVMHAChecker command [{}] returned [{}]. It may cause a shutdown of the host.", hostIp, cmd.toString(), parsedLine); + logger.warn("Checking heart beat for host IP {} with KVMHAChecker command [{}] returned [{}]. The host is considered dead, which may trigger HA actions, such as fencing it and restarting its VMs on other hosts.", hostIp, cmd.toString(), parsedLine); return false; } else { logger.debug("Checking heart beat for host IP {} with KVMHAChecker command [{}] succeeded.", hostIp, cmd.toString()); @@ -402,22 +509,27 @@ public Boolean hasHeartBeat(HAStoragePool pool, HostTO host) { public Boolean hasVmActivity(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration) { String hostIp = host.getPrivateNetwork().getIp(); Script cmd = new Script(vmActivityCheckPath, activityScriptTimeout.getStandardSeconds(), logger); - cmd.add("-i", pool.getPoolIp()); - cmd.add("-p", pool.getPoolMountSourcePath()); - cmd.add("-m", pool.getMountDestPath()); + addPoolConnectionArgs(cmd, pool); cmd.add("-h", hostIp); - cmd.add("-u", volumeUUIDListString); + File volumeListFile = addVolumeListArgs(cmd, volumeUUIDListString); cmd.add("-t", String.valueOf(System.currentTimeMillis() / 1000)); cmd.add("-d", String.valueOf(duration)); OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); - String result = cmd.execute(parser); + String result; + try { + result = cmd.execute(parser, getHeartBeatScriptEnvironment()); + } finally { + if (volumeListFile != null && !volumeListFile.delete()) { + logger.debug("Unable to delete the file {}", volumeListFile); + } + } String parsedLine = parser.getLine(); logger.debug("Checking VM activity for host IP {} with KVMHAVMActivityChecker [{command=\"{}\", result: \"{}\", log: \"{}\", pool: \"{}\"}].", hostIp, cmd.toString(), result, parsedLine, pool.getPoolIp()); if (result == null && parsedLine.contains("DEAD")) { - logger.warn("Checking VM activity for host IP {} with KVMHAVMActivityChecker command [{}] returned [{}]. It may cause a shutdown of the host.", hostIp, cmd.toString(), parsedLine); + logger.warn("Checking VM activity for host IP {} with KVMHAVMActivityChecker command [{}] returned [{}]. The host is considered dead, which may trigger HA actions, such as fencing it and restarting its VMs on other hosts.", hostIp, cmd.toString(), parsedLine); return false; } else { logger.debug("Checking VM activity for host IP {} with KVMHAVMActivityChecker command [{}] succeeded.", hostIp, cmd.toString()); diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapperTest.java new file mode 100644 index 000000000000..50e7427aabd6 --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapperTest.java @@ -0,0 +1,157 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.joda.time.Duration; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.CheckVMActivityOnStoragePoolCommand; +import com.cloud.agent.api.to.HostTO; +import com.cloud.agent.api.to.StorageFilerTO; +import com.cloud.hypervisor.kvm.resource.KVMHABase; +import com.cloud.hypervisor.kvm.resource.KVMHAMonitor; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.utils.script.Script; + +@RunWith(MockitoJUnitRunner.class) +public class LibvirtCheckVMActivityOnStoragePoolCommandWrapperTest { + + private static final String POOL_UUID = "7684f6e5-fd91-3435-b04e-59ae0efb3914"; + private static final String RBD_SCRIPT = "/scripts/kvmvmactivity_rbd.py"; + private static final String SCRIPT = "/scripts/kvmvmactivity.sh"; + + private final LibvirtCheckVMActivityOnStoragePoolCommandWrapper wrapper = new LibvirtCheckVMActivityOnStoragePoolCommandWrapper(); + private final LibvirtComputingResource resource = mock(LibvirtComputingResource.class); + private final KVMStoragePool pool = mock(KVMStoragePool.class); + private final CheckVMActivityOnStoragePoolCommand command = mock(CheckVMActivityOnStoragePoolCommand.class); + + private void setUp(StoragePoolType type, boolean supportsHa) { + StorageFilerTO filer = mock(StorageFilerTO.class); + when(filer.getType()).thenReturn(type); + when(filer.getUuid()).thenReturn(POOL_UUID); + when(command.getPool()).thenReturn(filer); + KVMStoragePoolManager poolManager = mock(KVMStoragePoolManager.class); + when(poolManager.getStoragePool(type, POOL_UUID)).thenReturn(pool); + when(resource.getStoragePoolMgr()).thenReturn(poolManager); + when(pool.isPoolSupportHA()).thenReturn(supportsHa); + } + + private KVMHABase.HAStoragePool addHaPool() { + KVMHABase.HAStoragePool haPool = new KVMHABase.HAStoragePool(pool, "10.0.0.1", "cloudstack", KVMHABase.PoolType.PrimaryStorage); + KVMHAMonitor monitor = mock(KVMHAMonitor.class); + when(monitor.getStoragePool(POOL_UUID)).thenReturn(haPool); + when(resource.getMonitor()).thenReturn(monitor); + return haPool; + } + + private void expectActivityCheck(String volumes, long suspectTime, boolean activity) { + when(command.getHost()).thenReturn(mock(HostTO.class)); + when(command.getVolumeList()).thenReturn(volumes); + when(command.getSuspectTimeInSeconds()).thenReturn(suspectTime); + when(pool.hasVmActivity(any(), any(), any(), anyString(), anyString(), anyLong())).thenReturn(activity); + } + + @Test + public void testRbdPoolIsCheckedWithTheScriptOfRbd() { + setUp(StoragePoolType.RBD, true); + KVMHABase.HAStoragePool haPool = addHaPool(); + expectActivityCheck("volume-1,volume-2", 123L, false); + + try (MockedStatic