Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ public int expungeByVmList(List<Long> vmIds, Long batchSize) {
public int getErroredInstanceCount(long vmGroupId) {
SearchCriteria<Integer> sc = CountBy.create();
sc.setParameters("vmGroupId", vmGroupId);
sc.setJoinParameters("vmSearch", "states", State.Error);
sc.setJoinParameters("vmSearch", "states", State.Error, State.Stopped);
final List<Integer> results = customSearch(sc, null);
return results.get(0);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,20 @@ public void testCountAvailableVmsByGroup() throws Exception {
Mockito.verify(searchCriteriaCountAvailableVmsByGroup).setJoinParameters("vmSearch", "states", new Object[] {VirtualMachine.State.Starting, VirtualMachine.State.Running, VirtualMachine.State.Stopping, VirtualMachine.State.Migrating});
}

@Test
public void testGetErroredInstanceCount() throws Exception {
Mockito.doReturn(Arrays.asList(3)).when(AutoScaleVmGroupVmMapDaoImplSpy).customSearch(Mockito.any(SearchCriteria.class), Mockito.eq(null));

long groupId = 4L;

int result = AutoScaleVmGroupVmMapDaoImplSpy.getErroredInstanceCount(groupId);

Assert.assertEquals(3, result);

Mockito.verify(searchCriteriaCountAvailableVmsByGroup).setParameters("vmGroupId", groupId);
Mockito.verify(searchCriteriaCountAvailableVmsByGroup).setJoinParameters("vmSearch", "states", new Object[] {VirtualMachine.State.Error, VirtualMachine.State.Stopped});
}

@Test
public void testCountByGroup() throws Exception {
Mockito.doNothing().when(searchCriteriaAutoScaleVmGroupVmMapVOMock).setParameters(Mockito.anyString(), Mockito.any());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2112,7 +2112,7 @@ public void doScaleUp(long groupId, Integer numVm) {
String.format("Failed to assign LB rule for VM %s in AutoScale VM group %s", vm, asGroup), groupId, ApiCommandResourceType.AutoScaleVmGroup.toString(), 0);
break;
}
} catch (ServerApiException e) {
} catch (CloudRuntimeException e) {
logger.error("Can not deploy new VM for scaling up in the group {}. Waiting for next round", asGroup);
ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, asGroup.getAccountId(), EventVO.LEVEL_ERROR, EventTypes.EVENT_AUTOSCALEVMGROUP_SCALEUP,
String.format("Failed to start VM %s in AutoScale VM group %s", vm, asGroup), groupId, ApiCommandResourceType.AutoScaleVmGroup.toString(), 0);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1526,6 +1526,43 @@ public void testDoScaleUp() throws ResourceUnavailableException, InsufficientCap
}
}

/**
* Regression test for #14185: VirtualMachineManagerImpl.start() wraps a failed start into an unchecked
* CloudRuntimeException rather than the checked exceptions startNewVM converts to ServerApiException.
* Before the fix, doScaleUp's catch(ServerApiException) missed it, the VM was never destroyed, and its
* autoscale_vmgroup_vm_map row leaked forever (the VM stays in State.Stopped, invisible to both
* getErroredInstanceCount() and countAvailableVmsByGroup(), so the group scales up again next interval).
*/
@Test
public void testDoScaleUpDestroysVmWhenStartThrowsCloudRuntimeException() throws ResourceUnavailableException, InsufficientCapacityException, ResourceAllocationException {
try (MockedStatic<ActionEventUtils> ignored = Mockito.mockStatic(ActionEventUtils.class)) {
when(autoScaleVmGroupDao.findById(vmGroupId)).thenReturn(asVmGroupMock);
when(asVmGroupMock.getId()).thenReturn(vmGroupId);
when(asVmGroupMock.getAccountId()).thenReturn(accountId);
when(asVmGroupMock.getMaxMembers()).thenReturn(maxMembers);
when(autoScaleVmGroupVmMapDao.countAvailableVmsByGroup(vmGroupId)).thenReturn(maxMembers - 1);
when(autoScaleVmGroupVmMapDao.getErroredInstanceCount(vmGroupId)).thenReturn(0);
when(asVmGroupMock.getState()).thenReturn(AutoScaleVmGroup.State.ENABLED);

when(autoScaleVmGroupDao.updateState(vmGroupId, AutoScaleVmGroup.State.ENABLED, AutoScaleVmGroup.State.SCALING)).thenReturn(true);
when(autoScaleVmGroupDao.updateState(vmGroupId, AutoScaleVmGroup.State.SCALING, AutoScaleVmGroup.State.ENABLED)).thenReturn(true);
Mockito.doReturn(userVmMock).when(autoScaleManagerImplSpy).createNewVM(asVmGroupMock);
when(userVmMock.getId()).thenReturn(virtualMachineId);

Mockito.doThrow(new CloudRuntimeException(String.format("Unable to start a VM [%s] due to [Resource unavailable].", virtualMachineId)))
.when(userVmMgr).startVirtualMachine(virtualMachineId, null, new HashMap<>(), null);
Mockito.doReturn(true).when(autoScaleManagerImplSpy).destroyVm(virtualMachineId);

autoScaleManagerImplSpy.doScaleUp(vmGroupId, 1);

Mockito.verify(autoScaleVmGroupVmMapDao).persist(any(AutoScaleVmGroupVmMapVO.class));
Mockito.verify(autoScaleManagerImplSpy).destroyVm(virtualMachineId);
Mockito.verify(loadBalancingRulesService, Mockito.never()).assignToLoadBalancer(anyLong(), any(), any(), eq(true));
// the group must leave SCALING even though the start failed, so the next monitor interval can retry
Mockito.verify(autoScaleVmGroupDao).updateState(vmGroupId, AutoScaleVmGroup.State.SCALING, AutoScaleVmGroup.State.ENABLED);
}
}

@Test
public void testDoScaleDown() {
try (MockedStatic<ActionEventUtils> ignored = Mockito.mockStatic(ActionEventUtils.class)) {
Expand Down
Loading