fix: lock auto tune rollback before source lookup

Junjie

2026-04-27 04dc01cd25ea66f845ebec183b2399f056a61118

fix: lock auto tune rollback before source lookup

 src/main/java/com/zy/ai/service/impl/AutoTuneApplyServiceImpl.java

@@ -198,19 +198,18 @@
        Date now = new Date();
        AiAutoTuneJob rollbackJob = createRollbackJob(reason, now);

        List<AiAutoTuneChange> sourceChanges = findLatestSuccessfulChanges();
        if (sourceChanges.isEmpty()) {
            persistNoRollbackSourceJobInTransaction(rollbackJob, now);
            return buildResult(rollbackJob, new ArrayList<>(), false);
        }

        String lockKey = RedisKeyType.AI_AUTO_TUNE_APPLY_LOCK.key;
        String lockToken = UUID.randomUUID().toString();
        if (!redisUtil.trySetStringIfAbsent(lockKey, lockToken, APPLY_LOCK_SECONDS)) {
            return rejectRollbackForUnavailableLock(reason, sourceChanges, now, lockKey);
            return rejectRollbackForUnavailableLock(reason, now, lockKey);
        }

        try {
            List<AiAutoTuneChange> sourceChanges = findLatestSuccessfulChanges();
            if (sourceChanges.isEmpty()) {
                persistNoRollbackSourceJobInTransaction(rollbackJob, now);
                return buildResult(rollbackJob, new ArrayList<>(), false);
            }
            try {
                RollbackPersistenceResult persistenceResult = persistRollbackResultInTransaction(
                        rollbackJob,
@@ -236,16 +235,13 @@
    }

    private AutoTuneApplyResult rejectRollbackForUnavailableLock(String reason,
                                                                 List<AiAutoTuneChange> sourceChanges,
                                                                 Date now,
                                                                 String lockKey) {
        boolean lockKeyExists = redisUtil.hasKey(lockKey);
        LOGGER.warn("申请AI自动调参 rollback 锁失败，lockKey={}, lockKeyExists={}", lockKey, lockKeyExists);
        AiAutoTuneJob rollbackJob = createRollbackJob(reason, now);
        RollbackPersistenceResult persistenceResult = persistFailedRollbackResultInTransaction(
        RollbackPersistenceResult persistenceResult = persistRollbackLockFailureInTransaction(
                rollbackJob,
                sourceChanges,
                new IllegalStateException(APPLY_LOCK_BUSY_REASON),
                now
        );
        return buildResult(rollbackJob, persistenceResult.getRollbackChanges(), false);
@@ -542,6 +538,18 @@
        });
    }

    private RollbackPersistenceResult persistRollbackLockFailureInTransaction(AiAutoTuneJob rollbackJob, Date now) {
        TransactionTemplate transactionTemplate = new TransactionTemplate(transactionManager);
        return transactionTemplate.execute(status -> {
            saveJob(rollbackJob);
            List<AiAutoTuneChange> rollbackChanges = buildRollbackLockFailureChanges(rollbackJob.getId(), now);
            saveAuditChanges(rollbackChanges);
            finishRollbackJob(rollbackJob, rollbackChanges, now);
            updateJob(rollbackJob);
            return new RollbackPersistenceResult(rollbackChanges, false);
        });
    }

    private RollbackPersistenceResult rollbackChanges(Long rollbackJobId, List<AiAutoTuneChange> sourceChanges, Date now) {
        List<AiAutoTuneChange> rollbackChanges = new ArrayList<>();
        boolean refreshConfigCache = false;
@@ -564,6 +572,18 @@
        return new RollbackPersistenceResult(rollbackChanges, refreshConfigCache);
    }

    private List<AiAutoTuneChange> buildRollbackLockFailureChanges(Long rollbackJobId, Date now) {
        AiAutoTuneChange rollbackChange = new AiAutoTuneChange();
        rollbackChange.setJobId(rollbackJobId);
        rollbackChange.setTargetType(AutoTuneTriggerType.ROLLBACK.getCode());
        rollbackChange.setTargetId("");
        rollbackChange.setTargetKey("latest_successful_job");
        rollbackChange.setResultStatus(ChangeStatus.FAILED.getCode());
        rollbackChange.setRejectReason(APPLY_LOCK_BUSY_REASON);
        rollbackChange.setCreateTime(now);
        return List.of(rollbackChange);
    }

    private void refreshRollbackConfigCacheIfNeeded(RollbackPersistenceResult persistenceResult) {
        if (persistenceResult != null && persistenceResult.isRefreshConfigCache()) {
            configService.refreshSystemConfigCache();

 src/test/java/com/zy/ai/service/AutoTuneApplyServiceImplTest.java

@@ -23,6 +23,7 @@
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.InOrder;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.mockito.junit.jupiter.MockitoSettings;
@@ -49,6 +50,7 @@
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -430,6 +432,9 @@
        verify(redisUtil).compareAndDelete(eq(RedisKeyType.AI_AUTO_TUNE_APPLY_LOCK.key), anyString());
        assertEquals(0, transactionManager.getJobSaveOutsideTransactionCount());
        assertEquals(0, transactionManager.getJobUpdateOutsideTransactionCount());
        InOrder rollbackOrder = inOrder(redisUtil, aiAutoTuneChangeService);
        rollbackOrder.verify(redisUtil).trySetStringIfAbsent(anyString(), anyString(), anyLong());
        rollbackOrder.verify(aiAutoTuneChangeService, atLeastOnce()).list(any(Wrapper.class));
    }

    @Test
@@ -494,12 +499,6 @@

    @Test
    void rollbackLockNotAcquiredReturnsFailedAuditWithoutTargetWrite() {
        AiAutoTuneJob latestRealJob = job(10L, "manual", "success");
        AiAutoTuneChange configChange = successChange(10L, "sys_config", "", "conveyorStationTaskLimit", "10", "15");
        when(aiAutoTuneChangeService.list(any(Wrapper.class)))
                .thenReturn(List.of(configChange))
                .thenReturn(List.of(configChange));
        when(aiAutoTuneJobService.getById(10L)).thenReturn(latestRealJob);
        when(redisUtil.trySetStringIfAbsent(anyString(), anyString(), anyLong())).thenReturn(false);

        AutoTuneApplyResult result = service.rollbackLastSuccessfulJob("manual rollback");
@@ -515,6 +514,8 @@
        assertTrue(transactionManager.getJobUpdateInsideTransactionCount() > 0);
        assertEquals(0, transactionManager.getJobSaveOutsideTransactionCount());
        assertEquals(0, transactionManager.getJobUpdateOutsideTransactionCount());
        verify(aiAutoTuneChangeService, never()).list(any(Wrapper.class));
        verify(aiAutoTuneJobService, never()).getById(any());
        verify(configService, never()).saveConfigValue(any(), any());
        verify(configService, never()).refreshSystemConfigCache();
        verify(redisUtil).hasKey(RedisKeyType.AI_AUTO_TUNE_APPLY_LOCK.key);