diff --git a/engine/schema/src/main/java/com/cloud/event/dao/EventDao.java b/engine/schema/src/main/java/com/cloud/event/dao/EventDao.java index c50451b03e4a..03716237cdb0 100644 --- a/engine/schema/src/main/java/com/cloud/event/dao/EventDao.java +++ b/engine/schema/src/main/java/com/cloud/event/dao/EventDao.java @@ -19,6 +19,7 @@ import java.util.Date; import java.util.List; +import com.cloud.event.Event; import com.cloud.event.EventVO; import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDao; @@ -31,6 +32,18 @@ public interface EventDao extends GenericDao { EventVO findCompletedEvent(long startId); + /** + * Finds the last non-archived start event matching the specified criteria. + * Events are ordered by ID in descending order, returning the most recent one. + * + * @param type the event type to search for + * @param state the event state to search for (e.g., {@link Event.State#Scheduled}) + * @param resourceId the resource ID associated with the event + * @param resourceType the resource type associated with the event + * @return the most recent EventVO matching the criteria, or null if not found + */ + EventVO findLastEvent(String type, Event.State state, Long resourceId, String resourceType); + public List listToArchiveOrDeleteEvents(List ids, String type, Date startDate, Date endDate, List accountIds); public void archiveEvents(List events); diff --git a/engine/schema/src/main/java/com/cloud/event/dao/EventDaoImpl.java b/engine/schema/src/main/java/com/cloud/event/dao/EventDaoImpl.java index e748e98900eb..b66da14292ef 100644 --- a/engine/schema/src/main/java/com/cloud/event/dao/EventDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/event/dao/EventDaoImpl.java @@ -19,7 +19,6 @@ import java.util.Date; import java.util.List; - import org.springframework.stereotype.Component; import com.cloud.event.Event.State; @@ -35,6 +34,7 @@ public class EventDaoImpl extends GenericDaoBase implements EventDao { protected final SearchBuilder CompletedEventSearch; protected final SearchBuilder ToArchiveOrDeleteEventSearch; + protected final SearchBuilder LastStartEventSearch; public EventDaoImpl() { CompletedEventSearch = createSearchBuilder(); @@ -51,6 +51,14 @@ public EventDaoImpl() { ToArchiveOrDeleteEventSearch.and("createdDateL", ToArchiveOrDeleteEventSearch.entity().getCreateDate(), Op.LTEQ); ToArchiveOrDeleteEventSearch.and("archived", ToArchiveOrDeleteEventSearch.entity().getArchived(), Op.EQ); ToArchiveOrDeleteEventSearch.done(); + + LastStartEventSearch = createSearchBuilder(); + LastStartEventSearch.and("type", LastStartEventSearch.entity().getType(), Op.EQ); + LastStartEventSearch.and("state", LastStartEventSearch.entity().getState(), Op.EQ); + LastStartEventSearch.and("resourceId", LastStartEventSearch.entity().getResourceId(), Op.EQ); + LastStartEventSearch.and("resourceType", LastStartEventSearch.entity().getResourceType(), Op.EQ); + LastStartEventSearch.and("archived", LastStartEventSearch.entity().getArchived(), Op.EQ); + LastStartEventSearch.done(); } @Override @@ -77,6 +85,17 @@ public EventVO findCompletedEvent(long startId) { return findOneIncludingRemovedBy(sc); } + @Override + public EventVO findLastEvent(String type, State state, Long resourceId, String resourceType) { + SearchCriteria sc = LastStartEventSearch.create(); + sc.setParameters("type", type); + sc.setParameters("state", state); + sc.setParameters("resourceId", resourceId); + sc.setParameters("resourceType", resourceType); + sc.setParameters("archived", false); + return findLastOneBy(sc); + } + @Override public List listToArchiveOrDeleteEvents(List ids, String type, Date startDate, Date endDate, List accountIds) { SearchCriteria sc = ToArchiveOrDeleteEventSearch.create(); diff --git a/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java b/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java index dcd863465d1b..dad1877adbc8 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java +++ b/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java @@ -929,7 +929,7 @@ public Class getEntityBeanType() { protected T findOneIncludingRemovedBy(final SearchCriteria sc) { Filter filter = new Filter(1, true); List results = searchIncludingRemoved(sc, filter, null, false); - assert results.size() <= 1 : "Didn't the limiting worked?"; + assert results.size() <= 1 : "Didn't the limiting work?"; return results.size() == 0 ? null : results.get(0); } @@ -949,6 +949,15 @@ public T findOneBy(SearchCriteria sc, final Filter filter) { return results.isEmpty() ? null : results.get(0); } + @DB() + protected T findLastOneBy(SearchCriteria sc) { + sc = checkAndSetRemovedIsNull(sc); + Filter filter = new Filter(_entityBeanType, "id", Boolean.FALSE, 0L, 1L); + List results = searchIncludingRemoved(sc, filter, null, false); + assert results.size() <= 1 : "Didn't the limiting work?"; + return results.size() == 0 ? null : results.get(0); + } + @DB() public List listBy(SearchCriteria sc, final Filter filter) { sc = checkAndSetRemovedIsNull(sc); diff --git a/server/src/main/java/com/cloud/event/ActionEventUtils.java b/server/src/main/java/com/cloud/event/ActionEventUtils.java index ae77446a8561..0ebb266fd7c2 100644 --- a/server/src/main/java/com/cloud/event/ActionEventUtils.java +++ b/server/src/main/java/com/cloud/event/ActionEventUtils.java @@ -400,6 +400,20 @@ private static long getDomainId(long accountId) { return account.getDomainId(); } + /** + * Retrieves the last non-archived event matching the specified criteria. + * + * @param type the event type to search for + * @param state the event state to search for (e.g., {@link Event.State#Scheduled}) + * @param resourceId the resource ID associated with the event + * @param resourceType the resource type associated with the event + * @return the most recent EventVO matching the criteria, or null if not found + * @see EventDao#findLastEvent(String, Event.State, Long, String) + */ + public static EventVO getLastEvent(String type, Event.State state, Long resourceId, String resourceType) { + return s_eventDao.findLastEvent(type, state, resourceId, resourceType); + } + private static void populateFirstClassEntities(Map eventDescription){ CallContext context = CallContext.current(); diff --git a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java index 755de00dec26..66ab4b9ffeeb 100644 --- a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java +++ b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java @@ -17,6 +17,7 @@ package com.cloud.ha; import static org.apache.cloudstack.framework.config.ConfigKey.Scope.Zone; +import static com.cloud.event.Event.State; import java.util.ArrayList; import java.util.Arrays; @@ -32,8 +33,14 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; -import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.context.CallContext; +import com.cloud.event.ActionEventUtils; +import com.cloud.event.Event; +import com.cloud.event.EventTypes; +import com.cloud.event.EventVO; +import com.cloud.user.Account; +import com.cloud.user.User; +import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider; @@ -452,9 +459,16 @@ public boolean scheduleMigration(final VMInstanceVO vm, HighAvailabilityManager. } Long hostId = VirtualMachine.State.Migrating.equals(vm.getState()) ? vm.getLastHostId() : vm.getHostId(); - final HaWorkVO work = new HaWorkVO(vm.getId(), vm.getType(), WorkType.Migration, Step.Scheduled, vm.getHostId(), vm.getState(), 0, vm.getUpdated(), reasonType); + final HaWorkVO work = new HaWorkVO(vm.getId(), vm.getType(), WorkType.Migration, Step.Scheduled, hostId, vm.getState(), 0, vm.getUpdated(), reasonType); _haDao.persist(work); - logger.info("Scheduled migration work of VM {} from host {} with HAWork {}", vm, _hostDao.findById(vm.getHostId()), work); + + HostVO host = _hostDao.findById(hostId); + logger.info(String.format("Scheduled migration work of VM %s from host %s with HAWork %s", vm, host, work)); + String hostName = Optional.ofNullable(host).map(HostVO::getName).orElse("N/A"); + String msg = String.format("Scheduled migration work of VM %s from host %s (%s) with HAWork %s (attempt %s of %s)", + vm.getHostName(), hostId, hostName, work.getId(), work.getTimesTried() + 1, _maxRetries); + createEvent(vm.getId(), ApiCommandResourceType.VirtualMachine, EventTypes.EVENT_VM_MIGRATE, msg, + State.Scheduled, EventVO.LEVEL_INFO); wakeupWorkers(); return true; } @@ -862,18 +876,77 @@ protected boolean checkAndCancelWorkIfNeeded(final HaWorkVO work) { return true; } + /** + * Creates an event for {@link ApiCommandResourceType} operations. + * This is a fail-safe helper method for logging purposes - exceptions are caught and logged. + * + * @param resourceId the resource ID + * @param resourceType the event resource type ({@link ApiCommandResourceType}) + * @param type the event type ({@link EventTypes}) + * @param description the event description + * @param state the event state ({@link Event.State}) + * @param level the event level (e.g., {@link EventVO#LEVEL_INFO} or {@link EventVO#LEVEL_ERROR}) + */ + private void createEvent(Long resourceId, ApiCommandResourceType resourceType, String type, String description, + State state, String level) { + try { + String resourceTypeStr = resourceType.toString(); + Long userId = User.UID_SYSTEM; + Long accountId = Account.ACCOUNT_ID_SYSTEM; + if (ApiCommandResourceType.VirtualMachine.equals(resourceType) && resourceId != null) { + VMInstanceVO vm = _instanceDao.findById(resourceId); + if (vm != null) { + accountId = vm.getAccountId(); + } + } + long startEventId = state == State.Scheduled ? 0L + : Optional.ofNullable(ActionEventUtils.getLastEvent(type, State.Scheduled, resourceId, + resourceTypeStr)) + .map(EventVO::getId).orElse(0L); + + switch (state) { + case Started: + ActionEventUtils.onStartedActionEvent(userId, accountId, type, description, resourceId, + resourceTypeStr, true, startEventId); + break; + case Scheduled: + ActionEventUtils.onScheduledActionEvent(userId, accountId, type, description, resourceId, + resourceTypeStr, true, startEventId); + break; + case Completed: + ActionEventUtils.onCompletedActionEvent(userId, accountId, level, type, true, + description, resourceId, resourceTypeStr, startEventId); + break; + default: + throw new CloudRuntimeException("Unsupported event state: " + state); + } + } catch (Exception e) { + logger.error(String.format("Failed to create event for VM: %s, command: %s, state: %s, level: %s", + resourceId, type, state, level), e); + } + } + public Long migrate(final HaWorkVO work) { logger.debug("MIGRATE with HA WORK"); long vmId = work.getInstanceId(); long srcHostId = work.getHostId(); HostVO srcHost = _hostDao.findById(srcHostId); + ApiCommandResourceType resourceType = ApiCommandResourceType.VirtualMachine; + String eventType = EventTypes.EVENT_VM_MIGRATE; + int attemptNumber = work.getTimesTried() + 1; VMInstanceVO vm = _instanceDao.findById(vmId); if (vm == null) { - logger.info("Unable to find vm: {}, skipping migrate.", vmId); + String msg = String.format("Unable to find vm %s, skipping migration. HA Work %s (attempt %s of %s)", + vmId, work.getId(), attemptNumber, _maxRetries); + logger.info(msg); + createEvent(vmId, resourceType, eventType, msg, State.Completed, EventVO.LEVEL_ERROR); return null; } if (checkAndCancelWorkIfNeeded(work)) { + String msg = String.format("Cancelled migration for vm %s as it is not needed anymore. HA Work %s (attempt %s of %s)", + vm.getHostName(), work.getId(), attemptNumber, _maxRetries); + createEvent(vmId, resourceType, eventType, msg, State.Completed, EventVO.LEVEL_ERROR); return null; } logger.info("Migration attempt: for {} from {}. Starting attempt: {}/{} times.", vm, srcHost, 1 + work.getTimesTried(), _maxRetries); @@ -883,23 +956,41 @@ public Long migrate(final HaWorkVO work) { return null; } if (VirtualMachine.State.Running.equals(vm.getState()) && srcHostId != vm.getHostId()) { - logger.info("VM {} is running on a different host {}, skipping migration", vm, vm.getHostId()); + String vmHostName = Optional.ofNullable(_hostDao.findById(vm.getHostId())).map(HostVO::getName) + .orElse("N/A"); + String msg = String.format("VM %s is running on a different host (%s), skipping migration. HA Work %s (attempt %s of %s)", + vm.getHostName(), vmHostName, work.getId(), attemptNumber, _maxRetries); + logger.info(msg); + createEvent(vmId, resourceType, eventType, msg, State.Completed, EventVO.LEVEL_ERROR); return null; } - + logger.info(String.format("Migration attempt: for VM %s from host %s. Starting attempt: %d/%d times.", + vm, srcHost, attemptNumber, _maxRetries)); try { + String vmHostName = Optional.ofNullable(_hostDao.findById(vm.getHostId())).map(HostVO::getName) + .orElse("N/A"); + String msg = String.format("Starting migration from host %s. HA Work %s (attempt %s of %s)", + vmHostName, work.getId(), attemptNumber, _maxRetries); + createEvent(vmId, resourceType, eventType, msg, State.Started, EventVO.LEVEL_INFO); work.setStep(Step.Migrating); _haDao.update(work.getId(), work); - // First try starting the vm with its original planner, if it doesn't succeed send HAPlanner as its an emergency. _itMgr.migrateAway(vm.getUuid(), srcHostId); + msg = String.format("Completed migration. HA Work %s (attempt %s of %s)", work.getId(), attemptNumber, _maxRetries); + createEvent(vmId, resourceType, eventType, msg, State.Completed, EventVO.LEVEL_INFO); return null; } catch (InsufficientServerCapacityException e) { - logger.warn("Migration attempt: Insufficient capacity for migrating a VM {} from source host {}. Exception: {}", vm, srcHost, e.getMessage()); + String msg = String.format("Migration attempt: Insufficient capacity for migrating a VM %s from source host %s. HA Work %s (attempt %s of %s)", + vm.getHostName(), srcHost, work.getId(), attemptNumber, _maxRetries); + logger.warn(msg); _resourceMgr.migrateAwayFailed(srcHostId, vmId); + createEvent(vmId, resourceType, eventType, msg, State.Completed, EventVO.LEVEL_ERROR); return (System.currentTimeMillis() >> 10) + _migrateRetryInterval; } catch (Exception e) { - logger.warn("Migration attempt: Unexpected exception occurred when attempting migration of {} {}", vm, e.getMessage()); + String msg = String.format("Migration attempt: Unexpected exception occurred when attempting migration of vm %s. HA Work %s (attempt %s of %s)", + vm.getHostName(), work.getId(), attemptNumber, _maxRetries); + logger.warn(msg); + createEvent(vmId, resourceType, eventType, msg, State.Completed, EventVO.LEVEL_ERROR); throw e; } } diff --git a/server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java b/server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java index 626f2cda172f..1fe4263afc59 100644 --- a/server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java +++ b/server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java @@ -78,6 +78,7 @@ import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.dao.VMInstanceDao; +import org.springframework.test.util.ReflectionTestUtils; @RunWith(MockitoJUnitRunner.class) public class HighAvailabilityManagerImplTest { @@ -309,7 +310,12 @@ public void scheduleMigration() { Mockito.when(vm.getType()).thenReturn(VirtualMachine.Type.User); Mockito.when(vm.getState()).thenReturn(VirtualMachine.State.Running); Mockito.when(vm.getHostId()).thenReturn(1L); - Mockito.when(_haDao.persist((HaWorkVO)Mockito.any())).thenReturn(Mockito.mock(HaWorkVO.class)); + + Mockito.when(_haDao.persist((HaWorkVO) Mockito.any())).thenAnswer(invocation -> { + HaWorkVO haWork = invocation.getArgument(0); + ReflectionTestUtils.setField(haWork, "id", 1L); + return haWork; + }); ConfigKey haEnabled = Mockito.mock(ConfigKey.class); highAvailabilityManager.VmHaEnabled = haEnabled;